| [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 |
|
Fetching ... files: 0it [00:00, ?it/s]
Fetching ... files: 7it [00:00, 11838.76it/s] |
| logs/b9ed9818-4ffe-4f5a-aff8-0df5b4a150d1.txt |
| Compiling model and warming up kernels (~7 minutes on first execution) |
| Sampling steps [0, 1, 4398, 4399, 4400, 4401, 8798, 8799, 8800, 8801, 13198, 13199, 13200, 13201] for warmup |
| W0615 01:45:46.376000 198186 torch/distributed/elastic/agent/server/api.py:739] Received 15 death signal, shutting down workers |
| W0615 01:45:46.376000 198186 torch/distributed/elastic/multiprocessing/api.py:1010] Sending process 198206 closing signal SIGTERM |
| W0615 01:45:46.377000 198186 torch/distributed/elastic/multiprocessing/api.py:1010] Sending process 198206 closing signal SIGTERM |
| Traceback (most recent call last): |
| File "/home/ubuntu/modded-nanogpt/.venv/lib/python3.14/site-packages/torch/distributed/elastic/agent/server/api.py", line 731, in run |
| result = self._invoke_run(role) |
| File "/home/ubuntu/modded-nanogpt/.venv/lib/python3.14/site-packages/torch/distributed/elastic/agent/server/api.py", line 908, in _invoke_run |
| time.sleep(monitor_interval) |
| ~~~~~~~~~~^^^^^^^^^^^^^^^^^^ |
| File "/home/ubuntu/modded-nanogpt/.venv/lib/python3.14/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 86, in _terminate_process_handler |
| raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) |
| torch.distributed.elastic.multiprocessing.api.SignalException: Process 198186 got signal: 15 |
|
|
| During handling of the above exception, another exception occurred: |
|
|
| Traceback (most recent call last): |
| File "/home/ubuntu/modded-nanogpt/.venv/bin/torchrun", line 6, in <module> |
| sys.exit(main()) |
| ~~~~^^ |
| File "/home/ubuntu/modded-nanogpt/.venv/lib/python3.14/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 362, in wrapper |
| return f(*args, **kwargs) |
| File "/home/ubuntu/modded-nanogpt/.venv/lib/python3.14/site-packages/torch/distributed/run.py", line 991, in main |
| run(args) |
| ~~~^^^^^^ |
| File "/home/ubuntu/modded-nanogpt/.venv/lib/python3.14/site-packages/torch/distributed/run.py", line 982, in run |
| elastic_launch( |
| ~~~~~~~~~~~~~~~ |
| config=config, |
| ~~~~~~~~~~~~~~ |
| entrypoint=cmd, |
| ~~~~~~~~~~~~~~~ |
| )(*cmd_args) |
| ~^^^^^^^^^^^ |
| File "/home/ubuntu/modded-nanogpt/.venv/lib/python3.14/site-packages/torch/distributed/launcher/api.py", line 170, in __call__ |
| return launch_agent(self._config, self._entrypoint, list(args)) |
| File "/home/ubuntu/modded-nanogpt/.venv/lib/python3.14/site-packages/torch/distributed/launcher/api.py", line 308, in launch_agent |
| result = agent.run() |
| File "/home/ubuntu/modded-nanogpt/.venv/lib/python3.14/site-packages/torch/distributed/elastic/metrics/api.py", line 134, in wrapper |
| result = f(*args, **kwargs) |
| File "/home/ubuntu/modded-nanogpt/.venv/lib/python3.14/site-packages/torch/distributed/elastic/agent/server/api.py", line 740, in run |
| self._shutdown(e.sigval) |
| ~~~~~~~~~~~~~~^^^^^^^^^^ |
| File "/home/ubuntu/modded-nanogpt/.venv/lib/python3.14/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 413, in _shutdown |
| self._pcontext.close(death_sig) |
| ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^ |
| File "/home/ubuntu/modded-nanogpt/.venv/lib/python3.14/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 659, in close |
| self._close(death_sig=death_sig, timeout=timeout) |
| ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ |
| File "/home/ubuntu/modded-nanogpt/.venv/lib/python3.14/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 1022, in _close |
| handler.proc.wait(time_to_wait) |
| ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^ |
| File "/usr/lib/python3.14/subprocess.py", line 1279, in wait |
| return self._wait(timeout=timeout) |
| ~~~~~~~~~~^^^^^^^^^^^^^^^^^ |
| File "/usr/lib/python3.14/subprocess.py", line 2078, in _wait |
| time.sleep(delay) |
| ~~~~~~~~~~^^^^^^^ |
| File "/home/ubuntu/modded-nanogpt/.venv/lib/python3.14/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 86, in _terminate_process_handler |
| raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) |
| torch.distributed.elastic.multiprocessing.api.SignalException: Process 198186 got signal: 1 |
|
|
