Upload Fast-SANA_1024_100k/debug_sana_connector_learnable.txt with huggingface_hub

Fast-SANA_1024_100k/debug_sana_connector_learnable.txt

@@ -0,0 +1,93 @@
+W0818 01:32:11.361000 22608120907584 torch/distributed/run.py:757] 
+W0818 01:32:11.361000 22608120907584 torch/distributed/run.py:757] *****************************************
+W0818 01:32:11.361000 22608120907584 torch/distributed/run.py:757] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0818 01:32:11.361000 22608120907584 torch/distributed/run.py:757] *****************************************
+/proj/cvl/users/x_fahkh2/envs/blip3o/bin/python: can't open file '/proj/cvl/users/x_fahkh2/BLIP3o_SANA/fastvlm-o/Fast-SANA/blip3o/train/train_mem.py': [Errno 2] No such file or directory
+/proj/cvl/users/x_fahkh2/envs/blip3o/bin/python: can't open file '/proj/cvl/users/x_fahkh2/BLIP3o_SANA/fastvlm-o/Fast-SANA/blip3o/train/train_mem.py': [Errno 2] No such file or directory
+/proj/cvl/users/x_fahkh2/envs/blip3o/bin/python: can't open file '/proj/cvl/users/x_fahkh2/BLIP3o_SANA/fastvlm-o/Fast-SANA/blip3o/train/train_mem.py': [Errno 2] No such file or directory
+/proj/cvl/users/x_fahkh2/envs/blip3o/bin/python: can't open file '/proj/cvl/users/x_fahkh2/BLIP3o_SANA/fastvlm-o/Fast-SANA/blip3o/train/train_mem.py': [Errno 2] No such file or directory
+/proj/cvl/users/x_fahkh2/envs/blip3o/bin/python: can't open file '/proj/cvl/users/x_fahkh2/BLIP3o_SANA/fastvlm-o/Fast-SANA/blip3o/train/train_mem.py': [Errno 2] No such file or directory
+/proj/cvl/users/x_fahkh2/envs/blip3o/bin/python: can't open file '/proj/cvl/users/x_fahkh2/BLIP3o_SANA/fastvlm-o/Fast-SANA/blip3o/train/train_mem.py': [Errno 2] No such file or directory
+/proj/cvl/users/x_fahkh2/envs/blip3o/bin/python: can't open file '/proj/cvl/users/x_fahkh2/BLIP3o_SANA/fastvlm-o/Fast-SANA/blip3o/train/train_mem.py': [Errno 2] No such file or directory
+/proj/cvl/users/x_fahkh2/envs/blip3o/bin/python: can't open file '/proj/cvl/users/x_fahkh2/BLIP3o_SANA/fastvlm-o/Fast-SANA/blip3o/train/train_mem.py': [Errno 2] No such file or directory
+E0818 01:32:16.405000 22608120907584 torch/distributed/elastic/multiprocessing/api.py:826] failed (exitcode: 2) local_rank: 0 (pid: 862766) of binary: /proj/cvl/users/x_fahkh2/envs/blip3o/bin/python
+Traceback (most recent call last):
+  File "/proj/cvl/users/x_fahkh2/envs/blip3o/bin/torchrun", line 8, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/proj/cvl/users/x_fahkh2/envs/blip3o/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 347, in wrapper
+    return f(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^
+  File "/proj/cvl/users/x_fahkh2/envs/blip3o/lib/python3.11/site-packages/torch/distributed/run.py", line 879, in main
+    run(args)
+  File "/proj/cvl/users/x_fahkh2/envs/blip3o/lib/python3.11/site-packages/torch/distributed/run.py", line 870, in run
+    elastic_launch(
+  File "/proj/cvl/users/x_fahkh2/envs/blip3o/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 132, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/proj/cvl/users/x_fahkh2/envs/blip3o/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 263, in launch_agent
+    raise ChildFailedError(
+torch.distributed.elastic.multiprocessing.errors.ChildFailedError: 
+============================================================
+blip3o/train/train_mem.py FAILED
+------------------------------------------------------------
+Failures:
+[1]:
+  time      : 2025-08-18_01:32:16
+  host      : node045
+  rank      : 1 (local_rank: 1)
+  exitcode  : 2 (pid: 862767)
+  error_file: <N/A>
+  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
+[2]:
+  time      : 2025-08-18_01:32:16
+  host      : node045
+  rank      : 2 (local_rank: 2)
+  exitcode  : 2 (pid: 862768)
+  error_file: <N/A>
+  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
+[3]:
+  time      : 2025-08-18_01:32:16
+  host      : node045
+  rank      : 3 (local_rank: 3)
+  exitcode  : 2 (pid: 862769)
+  error_file: <N/A>
+  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
+[4]:
+  time      : 2025-08-18_01:32:16
+  host      : node045
+  rank      : 4 (local_rank: 4)
+  exitcode  : 2 (pid: 862770)
+  error_file: <N/A>
+  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
+[5]:
+  time      : 2025-08-18_01:32:16
+  host      : node045
+  rank      : 5 (local_rank: 5)
+  exitcode  : 2 (pid: 862771)
+  error_file: <N/A>
+  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
+[6]:
+  time      : 2025-08-18_01:32:16
+  host      : node045
+  rank      : 6 (local_rank: 6)
+  exitcode  : 2 (pid: 862772)
+  error_file: <N/A>
+  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
+[7]:
+  time      : 2025-08-18_01:32:16
+  host      : node045
+  rank      : 7 (local_rank: 7)
+  exitcode  : 2 (pid: 862773)
+  error_file: <N/A>
+  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
+------------------------------------------------------------
+Root Cause (first observed failure):
+[0]:
+  time      : 2025-08-18_01:32:16
+  host      : node045
+  rank      : 0 (local_rank: 0)
+  exitcode  : 2 (pid: 862766)
+  error_file: <N/A>
+  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
+============================================================