diff --git "a/train.log" "b/train.log" new file mode 100644--- /dev/null +++ "b/train.log" @@ -0,0 +1,5040 @@ +[2024-08-22 00:33:02,521] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:02,781] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:02,887] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:02,907] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:02,992] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:03,142] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:03,382] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:03,521] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:04,012] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:04,012] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:04,012] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:04,012] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:04,074] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:04,297] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:04,297] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:04,374] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:04,374] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:04,409] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:04,502] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:04,503] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:04,548] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:04,654] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:04,744] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:04,781] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:05,565] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:05,617] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:05,759] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:05,764] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:05,765] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:05,950] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:05,989] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:06,247] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:06,263] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:06,281] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:06,430] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:06,435] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:06,542] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:06,663] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:06,880] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:07,066] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:07,752] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:07,752] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:07,752] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:07,758] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:07,767] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:07,925] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:08,056] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible +[2024-08-22 00:33:08,112] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible +[2024-08-22 00:33:08,159] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:08,160] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:08,271] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:08,313] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:08,371] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible +[2024-08-22 00:33:08,515] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:08,612] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:08,627] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible +[2024-08-22 00:33:09,032] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:09,032] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:09,032] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:09,116] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:09,151] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:09,242] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:09,255] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible +[2024-08-22 00:33:09,322] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 +[2024-08-22 00:33:09,919] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible +[2024-08-22 00:33:09,919] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible +[2024-08-22 00:33:09,970] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. +[2024-08-22 00:33:10,090] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH +[2024-08-22 00:33:10,138] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible +[2024-08-22 00:33:10,196] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. +[2024-08-22 00:33:10,274] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. +[2024-08-22 00:33:10,425] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible +[2024-08-22 00:33:11,391] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:11,391] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:11,417] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:11,417] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH +[2024-08-22 00:33:11,582] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible +[2024-08-22 00:33:11,589] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible +[2024-08-22 00:33:11,629] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:11,796] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible +[2024-08-22 00:33:12,732] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:12,737] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:12,797] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:12,888] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible +[2024-08-22 00:33:13,140] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:13,228] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:13,262] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible +[2024-08-22 00:33:13,344] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible +[2024-08-22 00:33:13,814] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:13,814] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:13,814] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:13,814] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:13,814] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible +[2024-08-22 00:33:14,024] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:14,035] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:33:14,038] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible +[2024-08-22 00:33:22,444] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:33:22,570] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:33:22,792] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:33:22,825] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:33:23,006] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:33:23,144] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:33:23,336] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:33:23,589] [INFO] [comm.py:637:init_distributed] cdb=None +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. + Loading checkpoint shards: 0%| | 0/4 [00:00 +hub_private_repo: False +hub_always_push: False +gradient_checkpointing: True +gradient_checkpointing_kwargs: None +include_inputs_for_metrics: False +eval_do_concat_batches: True +fp16_backend: auto +evaluation_strategy: steps +push_to_hub_model_id: None +push_to_hub_organization: None +push_to_hub_token: +mp_parameters: +auto_find_batch_size: False +full_determinism: False +torchdynamo: None +ray_scope: last +ddp_timeout: 1800 +torch_compile: False +torch_compile_backend: None +torch_compile_mode: None +dispatch_batches: None +split_batches: None +include_tokens_per_second: False +include_num_input_tokens_seen: False +neftune_noise_alpha: None +optim_target_modules: None +batch_eval_metrics: False +eval_on_start: False +eval_use_gather_object: False +data_root_dir: /fsx_0/user/imzyc/processed_data/ +train_datasets: ego4d/narration_train_L4096_I5,sthsthv2/narration_train_L4096_I5@10,llava/caption_train_L4096_I5@2,egoobjects/detection_train_L4096_I5@20,ego4d/summary_train_L4096_I5@2,holoassist/summary_train_L4096_I5@2,epickitchens/summary_train_L4096_I5@2,egoexolearn/summary_train_L4096_I5@2,wtag/summary_train_L4096_I5@2,assembly101/summary_train_L4096_I5@2,ego4d/dialog-klg-sum_train_L4096_I5@2,holoassist/dialog-klg-sum_train_L4096_I5@2,epickitchens/dialog-klg-sum_train_L4096_I5@2,egoexolearn/dialog-klg-sum_train_L4096_I5@2,wtag/dialog-klg-sum_train_L4096_I5@5,assembly101/dialog-klg-sum_train_L4096_I5@2 +eval_datasets: sthsthv2/narration_val_L4096_I5,ego4d/dialog-klg-sum_val_L4096_I5,holoassist/dialog-klg-sum_val_L4096_I5,ego4d/narration_val_L4096_I5,ego4d/summary_val_L4096_I5 +neg_frame_sampling_rate: 0.1 +use_pose: False +llm_train_mode: lora +lora_modules: model.*(q_proj|k_proj|v_proj|o_proj|gate_proj|up_proj|down_proj)|lm_head$ +lora_r: 128 +lora_alpha: 256 +finetune_modules: mm_projector +is_debug: False +**************************************************************************************************** + Loading checkpoint shards: 75%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 3/4 [00:00<00:00, 3.99it/s] Loading checkpoint shards: 50%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 2/4 [00:00<00:00, 6.32it/s] Loading checkpoint shards: 75%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 3/4 [00:00<00:00, 5.28it/s] Loading checkpoint shards: 25%|β–ˆβ–ˆβ–Œ | 1/4 [00:00<00:00, 4.18it/s] Loading checkpoint shards: 0%| | 0/4 [00:00= 1.5 and < 2.0 but detected 2.3 + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  async_io: please install the libaio-dev package with apt [WARNING]  async_io: please install the libaio-dev package with apt + + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  async_io: please install the libaio-dev package with apt [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible +[2024-08-22 00:48:44,063] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:48:44,063] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:48:44,063] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:48:44,063] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:48:44,063] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:48:44,063] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:48:44,063] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:48:44,063] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  async_io: please install the libaio-dev package with apt [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt [WARNING]  async_io: please install the libaio-dev package with apt + + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible +[2024-08-22 00:48:46,046] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,048] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,048] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,048] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,048] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,048] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,048] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,049] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,047] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,047] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,047] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,048] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,049] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,049] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,049] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,049] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,049] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,049] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,049] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,049] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,047] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,047] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,047] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,047] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,047] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,048] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,047] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,047] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,047] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,047] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,047] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,047] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,048] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,052] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,055] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,054] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,054] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,054] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,049] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,049] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,049] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,049] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,049] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,049] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,049] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,055] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,053] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,053] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,053] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,053] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,053] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,053] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,053] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,055] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,055] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,055] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,055] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,055] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,055] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,055] [INFO] [comm.py:668:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl +[2024-08-22 00:48:46,055] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,054] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,054] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,054] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,054] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,054] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,055] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,055] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,055] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,055] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,055] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,055] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,056] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,202] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,203] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,203] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,203] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,203] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,203] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,203] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,203] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,567] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,567] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,567] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,567] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,567] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,567] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,567] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 00:48:46,567] [INFO] [comm.py:637:init_distributed] cdb=None +**************************************** Experiment setup ****************************************: +llm_pretrained: meta-llama/Meta-Llama-3.1-8B-Instruct +vision_pretrained: google/siglip-so400m-patch14-384 +vision_hidden_size: 1152 +img_resolution: None +use_img_cls_token: True +img_patch_token_size: 2 +img_patch_token_layer: -2 +img_sep_token: +max_seq_len: 4096 +padding_side: right +attn_implementation: flash_attention_2 +w2t_logit_weight: 1.0 +use_binary_decision_head: False +binary_loss_weight: 0.1 +binary_decision_head_type: linear +output_dir: /fsx_0/user/imzyc/proact_exps/20240822-L4096-I5-ep4-NOSEP-nr0.1-klgmix-1s-lora-bs384-debug +overwrite_output_dir: False +do_train: False +do_eval: True +do_predict: False +eval_strategy: steps +prediction_loss_only: False +per_device_train_batch_size: 4 +per_device_eval_batch_size: 4 +per_gpu_train_batch_size: None +per_gpu_eval_batch_size: None +gradient_accumulation_steps: 1 +eval_accumulation_steps: None +eval_delay: 0 +torch_empty_cache_steps: None +learning_rate: 0.0002 +weight_decay: 0.0 +adam_beta1: 0.9 +adam_beta2: 0.999 +adam_epsilon: 1e-08 +max_grad_norm: 1.0 +num_train_epochs: 4.0 +max_steps: -1 +lr_scheduler_type: cosine +lr_scheduler_kwargs: {} +warmup_ratio: 0.0 +warmup_steps: 100 +log_level: passive +log_level_replica: warning +log_on_each_node: True +logging_dir: /fsx_0/user/imzyc/proact_exps/20240822-L4096-I5-ep4-NOSEP-nr0.1-klgmix-1s-lora-bs384-debug/runs/Aug22_00-48-42_h100-st-p548xlarge-37 +logging_strategy: steps +logging_first_step: False +logging_steps: 10 +logging_nan_inf_filter: True +save_strategy: epoch +save_steps: 500 +save_total_limit: 1 +save_safetensors: True +save_on_each_node: False +save_only_model: False +restore_callback_states_from_checkpoint: False +no_cuda: False +use_cpu: False +use_mps_device: False +seed: 42 +data_seed: None +jit_mode_eval: False +use_ipex: False +bf16: True +fp16: False +fp16_opt_level: O1 +half_precision_backend: auto +bf16_full_eval: False +fp16_full_eval: False +tf32: True +local_rank: 0 +ddp_backend: None +tpu_num_cores: None +tpu_metrics_debug: False +debug: [] +dataloader_drop_last: False +eval_steps: 400 +dataloader_num_workers: 2 +dataloader_prefetch_factor: 2 +past_index: -1 +run_name: 20240822-L4096-I5-ep4-NOSEP-nr0.1-klgmix-1s-lora-bs384-debug +disable_tqdm: False +remove_unused_columns: True +label_names: None +load_best_model_at_end: False +metric_for_best_model: None +greater_is_better: None +ignore_data_skip: False +fsdp: [] +fsdp_min_num_params: 0 +fsdp_config: {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False} +fsdp_transformer_layer_cls_to_wrap: None +accelerator_config: {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None} +deepspeed: deepspeed/zero2.json +label_smoothing_factor: 0.0 +optim: adamw_torch +optim_args: None +adafactor: False +group_by_length: False +length_column_name: length +report_to: ['tensorboard', 'wandb'] +ddp_find_unused_parameters: None +ddp_bucket_cap_mb: None +ddp_broadcast_buffers: None +dataloader_pin_memory: True +dataloader_persistent_workers: False +skip_memory_metrics: True +use_legacy_prediction_loop: False +push_to_hub: False +resume_from_checkpoint: None +hub_model_id: None +hub_strategy: every_save +hub_token: +hub_private_repo: False +hub_always_push: False +gradient_checkpointing: True +gradient_checkpointing_kwargs: None +include_inputs_for_metrics: False +eval_do_concat_batches: True +fp16_backend: auto +evaluation_strategy: steps +push_to_hub_model_id: None +push_to_hub_organization: None +push_to_hub_token: +mp_parameters: +auto_find_batch_size: False +full_determinism: False +torchdynamo: None +ray_scope: last +ddp_timeout: 1800 +torch_compile: False +torch_compile_backend: None +torch_compile_mode: None +dispatch_batches: None +split_batches: None +include_tokens_per_second: False +include_num_input_tokens_seen: False +neftune_noise_alpha: None +optim_target_modules: None +batch_eval_metrics: False +eval_on_start: False +eval_use_gather_object: False +data_root_dir: /fsx_0/user/imzyc/processed_data/ +train_datasets: ego4d/narration_train_L4096_I5,sthsthv2/narration_train_L4096_I5@10,llava/caption_train_L4096_I5@2,egoobjects/detection_train_L4096_I5@20,ego4d/summary_train_L4096_I5@2,holoassist/summary_train_L4096_I5@2,epickitchens/summary_train_L4096_I5@2,egoexolearn/summary_train_L4096_I5@2,wtag/summary_train_L4096_I5@2,assembly101/summary_train_L4096_I5@2,ego4d/dialog-klg-sum_train_L4096_I5@2,holoassist/dialog-klg-sum_train_L4096_I5@2,epickitchens/dialog-klg-sum_train_L4096_I5@2,egoexolearn/dialog-klg-sum_train_L4096_I5@2,wtag/dialog-klg-sum_train_L4096_I5@5,assembly101/dialog-klg-sum_train_L4096_I5@2 +eval_datasets: sthsthv2/narration_val_L4096_I5,ego4d/dialog-klg-sum_val_L4096_I5,holoassist/dialog-klg-sum_val_L4096_I5,ego4d/narration_val_L4096_I5,ego4d/summary_val_L4096_I5 +neg_frame_sampling_rate: 0.1 +use_pose: False +llm_train_mode: lora +lora_modules: model.*(q_proj|k_proj|v_proj|o_proj|gate_proj|up_proj|down_proj)|lm_head$ +lora_r: 128 +lora_alpha: 256 +finetune_modules: mm_projector +is_debug: False +**************************************************************************************************** +[2024-08-22 00:48:47,541] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:48:47,541] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:48:47,541] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:48:47,541] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:48:47,541] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:48:47,541] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:48:47,541] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 00:48:47,541] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. + Loading checkpoint shards: 0%| | 0/4 [00:00= 1.5 and < 2.0 but detected 2.3 + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. + Loading checkpoint shards: 0%| | 0/4 [00:00= 1.5 and < 2.0 but detected 2.3 + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. [WARNING]  async_io: please install the libaio-dev package with apt + + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  async_io: please install the libaio-dev package with apt [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  async_io: please install the libaio-dev package with apt [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt [WARNING]  async_io: please install the libaio-dev package with apt + + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible +[2024-08-22 02:03:55,771] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 02:03:55,771] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 02:03:55,771] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 02:03:55,771] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 02:03:55,771] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 02:03:55,771] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 02:03:55,771] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 02:03:55,772] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible +[2024-08-22 02:03:56,270] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 02:03:56,270] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 02:03:56,270] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 02:03:56,270] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 02:03:56,270] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 02:03:56,271] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 02:03:56,271] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 02:03:56,271] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt [WARNING]  async_io: please install the libaio-dev package with apt + + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. [WARNING]  async_io: please install the libaio-dev package with apt + + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible +[2024-08-22 02:03:56,803] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 02:03:56,803] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 02:03:56,803] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 02:03:56,803] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 02:03:56,804] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 02:03:56,804] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 02:03:56,804] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 02:03:56,804] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 02:03:56,925] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 02:03:56,926] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 02:03:56,926] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 02:03:56,926] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 02:03:56,926] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 02:03:56,926] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 02:03:56,926] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 02:03:56,927] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-08-22 02:03:56,994] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:56,994] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:56,994] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:56,994] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:56,994] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:56,994] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:56,994] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:56,994] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:56,997] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:56,997] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:56,997] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:56,997] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:56,997] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:56,998] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:56,998] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,001] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,004] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,004] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,004] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,004] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,004] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,004] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,004] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,025] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,097] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,097] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,097] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,097] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,097] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,097] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,097] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,097] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,125] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,125] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,125] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,125] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,125] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,125] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,125] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,125] [INFO] [comm.py:637:init_distributed] cdb=None + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH [WARNING]  async_io: please install the libaio-dev package with apt + + + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible +[2024-08-22 02:03:57,228] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,229] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,229] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,229] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,229] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,229] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,230] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,230] [INFO] [comm.py:637:init_distributed] cdb=None + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  async_io: please install the libaio-dev package with apt [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible +[2024-08-22 02:03:57,702] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,704] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,704] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,704] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,704] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,704] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,704] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,704] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,723] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,723] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,723] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,723] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,723] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,723] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,723] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:57,723] [INFO] [comm.py:637:init_distributed] cdb=None + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io requires the dev libaio .so object and headers but these were not found. + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  async_io: please install the libaio-dev package with apt [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  async_io: please install the libaio-dev package with apt + [WARNING]  If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found. + [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible + [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3 + [WARNING]  using untested triton version (2.3.1), only 1.0.0 is known to be compatible +[2024-08-22 02:03:58,258] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:58,260] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:58,260] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:58,260] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:58,260] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:58,260] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:58,260] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:58,260] [INFO] [comm.py:637:init_distributed] cdb=None +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +[2024-08-22 02:03:58,956] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:58,957] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:58,957] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:58,957] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:58,957] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:58,957] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:58,957] [INFO] [comm.py:637:init_distributed] cdb=None +[2024-08-22 02:03:58,957] [INFO] [comm.py:637:init_distributed] cdb=None +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. + Loading checkpoint shards: 0%| | 0/4 [00:00 +hub_private_repo: False +hub_always_push: False +gradient_checkpointing: True +gradient_checkpointing_kwargs: None +include_inputs_for_metrics: False +eval_do_concat_batches: True +fp16_backend: auto +evaluation_strategy: steps +push_to_hub_model_id: None +push_to_hub_organization: None +push_to_hub_token: +mp_parameters: +auto_find_batch_size: False +full_determinism: False +torchdynamo: None +ray_scope: last +ddp_timeout: 1800 +torch_compile: False +torch_compile_backend: None +torch_compile_mode: None +dispatch_batches: None +split_batches: None +include_tokens_per_second: False +include_num_input_tokens_seen: False +neftune_noise_alpha: None +optim_target_modules: None +batch_eval_metrics: False +eval_on_start: False +eval_use_gather_object: False +data_root_dir: /fsx_0/user/imzyc/processed_data/ +train_datasets: ego4d/narration_train_L4096_I5,sthsthv2/narration_train_L4096_I5@10,llava/caption_train_L4096_I5@2,egoobjects/detection_train_L4096_I5@20,ego4d/summary_train_L4096_I5@2,holoassist/summary_train_L4096_I5@2,epickitchens/summary_train_L4096_I5@2,egoexolearn/summary_train_L4096_I5@2,wtag/summary_train_L4096_I5@2,assembly101/summary_train_L4096_I5@2,ego4d/dialog-klg-sum_train_L4096_I5@2,holoassist/dialog-klg-sum_train_L4096_I5@2,epickitchens/dialog-klg-sum_train_L4096_I5@2,egoexolearn/dialog-klg-sum_train_L4096_I5@2,wtag/dialog-klg-sum_train_L4096_I5@5,assembly101/dialog-klg-sum_train_L4096_I5@2 +eval_datasets: sthsthv2/narration_val_L4096_I5,ego4d/dialog-klg-sum_val_L4096_I5,holoassist/dialog-klg-sum_val_L4096_I5,ego4d/narration_val_L4096_I5,ego4d/summary_val_L4096_I5 +neg_frame_sampling_rate: 0.1 +use_pose: False +llm_train_mode: lora +lora_modules: model.*(q_proj|k_proj|v_proj|o_proj|gate_proj|up_proj|down_proj)|lm_head$ +lora_r: 128 +lora_alpha: 256 +finetune_modules: mm_projector +is_debug: False +**************************************************************************************************** + Loading checkpoint shards: 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 4/4 [00:00<00:00, 5.92it/s] + Loading checkpoint shards: 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 4/4 [00:00<00:00, 6.48it/s] Loading checkpoint shards: 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 4/4 [00:00<00:00, 6.27it/s] + Loading checkpoint shards: 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 4/4 [00:00<00:00, 6.29it/s] Loading checkpoint shards: 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 4/4 [00:00<00:00, 6.18it/s] + Loading checkpoint shards: 50%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 2/4 [00:00<00:00, 6.45it/s] Loading checkpoint shards: 50%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 2/4 [00:00<00:00, 6.10it/s] Loading checkpoint shards: 50%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 2/4 [00:00<00:00, 6.20it/s] Loading checkpoint shards: 50%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 2/4 [00:00<00:00, 6.13it/s] Loading checkpoint shards: 50%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 2/4 [00:00<00:00, 6.28it/s] Loading checkpoint shards: 50%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 2/4 [00:00<00:00, 6.48it/s] Loading checkpoint shards: 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 4/4 [00:00<00:00, 5.36it/s] Loading checkpoint shards: 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 4/4 [00:00<00:00, 5.05it/s] + Loading checkpoint shards: 50%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 2/4 [00:00<00:00, 6.00it/s] Loading checkpoint shards: 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 4/4 [00:00<00:00, 6.66it/s] Loading checkpoint shards: 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 4/4 [00:00<00:00, 6.44it/s] +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. + Loading checkpoint shards: 75%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 3/4 [00:00<00:00, 6.55it/s] Loading checkpoint shards: 75%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 3/4 [00:00<00:00, 6.40it/s] Loading checkpoint shards: 75%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 3/4 [00:00<00:00, 6.39it/s] Loading checkpoint shards: 75%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 3/4 [00:00<00:00, 6.33it/s] Loading checkpoint shards: 75%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 3/4 [00:00<00:00, 6.45it/s] Loading checkpoint shards: 0%| | 0/4 [00:00