diff --git a/LTA_openwebtext_dualt/logs/lta_lm1b_classic_dirichlet_len1024_gbs512_8gpu_20k_save1k_20260523_watcher.sh b/LTA_openwebtext_dualt/logs/lta_lm1b_classic_dirichlet_len1024_gbs512_8gpu_20k_save1k_20260523_watcher.sh new file mode 100644 index 0000000000000000000000000000000000000000..7e54f00feb6c9b6e64f1532d743df9543739be37 --- /dev/null +++ b/LTA_openwebtext_dualt/logs/lta_lm1b_classic_dirichlet_len1024_gbs512_8gpu_20k_save1k_20260523_watcher.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt +export PYTHONPATH="$(pwd)${PYTHONPATH:+:$PYTHONPATH}" +export TOKENIZERS_PARALLELISM=false +export PYTHONUNBUFFERED=1 + +: "${RUN_DIR:?RUN_DIR is required}" +: "${OUT_BASE:?OUT_BASE is required}" +: "${LOG_DIR:?LOG_DIR is required}" +: "${TOKENIZER_PATH:?TOKENIZER_PATH is required}" +: "${SCORER:?SCORER is required}" + +RUN_STEM="$(basename "${RUN_DIR}")" +TEMP_TAG="${ENDPOINT_TEMP//./p}" +PROCESSED_FILE="${LOG_DIR}/processed_${RUN_STEM}_steps${STEPS}_c${CMAX}_t${TEMP_TAG}_n${N_SAMPLES}.txt" + +mkdir -p "${OUT_BASE}" "${LOG_DIR}" +touch "${PROCESSED_FILE}" + +echo "[watch-classic] run_dir=${RUN_DIR}" +echo "[watch-classic] out_base=${OUT_BASE}" +echo "[watch-classic] interval=${STEP_INTERVAL} max_len=${MAX_LEN} steps=${STEPS} cmax=${CMAX} temp=${ENDPOINT_TEMP} n=${N_SAMPLES}" + +while true; do + shopt -s nullglob + ckpts=("${RUN_DIR}"/step_*.pt) + shopt -u nullglob + + if (( ${#ckpts[@]} == 0 )); then + echo "[watch-classic] $(date +%F_%T) no ckpt yet" + sleep "${SLEEP_SECONDS}" + continue + fi + + printf "%s\n" "${ckpts[@]}" | sort | while read -r ckpt; do + base="$(basename "${ckpt}")" + step="${base#step_}" + step="${step%.pt}" + step_num=$((10#${step})) + + if (( step_num % STEP_INTERVAL != 0 )); then + continue + fi + if grep -Fxq "${ckpt}" "${PROCESSED_FILE}"; then + continue + fi + + out_dir="${OUT_BASE}/step_${step}" + log_file="${LOG_DIR}/infer_${RUN_STEM}_step_${step}.log" + mkdir -p "${out_dir}" + + echo "[watch-classic] $(date +%F_%T) infer ${ckpt} -> ${out_dir}" | tee -a "${log_file}" + CUDA_VISIBLE_DEVICES="${WATCH_CUDA_VISIBLE_DEVICES}" python scripts/eval_owt_normal_steps_sweep_20260515.py \ + --checkpoint "${ckpt}" \ + --tokenizer_path "${TOKENIZER_PATH}" \ + --scorer "${SCORER}" \ + --out_dir "${out_dir}" \ + --steps_list "${STEPS}" \ + --cmax_list "${CMAX}" \ + --endpoint_temps "${ENDPOINT_TEMP}" \ + --n_samples "${N_SAMPLES}" \ + --max_len "${MAX_LEN}" \ + --decode_batch "${DECODE_BATCH}" \ + --score_batch "${SCORE_BATCH}" \ + --score_max_length "${SCORE_MAX_LENGTH}" \ + --detokenizer lm1b \ + --seed 20260523 \ + --save_samples 16 \ + 2>&1 | tee -a "${log_file}" + + echo "${ckpt}" >> "${PROCESSED_FILE}" + echo "[watch-classic] $(date +%F_%T) done step_${step}" | tee -a "${log_file}" + done + + sleep "${SLEEP_SECONDS}" +done diff --git a/LTA_openwebtext_dualt/logs/lta_owt_dirichlet_categorical_fullvocab_c1024_fullycoupled_flmpack_onehot_hardce_ddit_small_len1024_gbs512_8gpu_1m_nw4_shufchunks.log b/LTA_openwebtext_dualt/logs/lta_owt_dirichlet_categorical_fullvocab_c1024_fullycoupled_flmpack_onehot_hardce_ddit_small_len1024_gbs512_8gpu_1m_nw4_shufchunks.log new file mode 100644 index 0000000000000000000000000000000000000000..0e3b086a7590cde972ac6e8d9bee2be23255ffb8 --- /dev/null +++ b/LTA_openwebtext_dualt/logs/lta_owt_dirichlet_categorical_fullvocab_c1024_fullycoupled_flmpack_onehot_hardce_ddit_small_len1024_gbs512_8gpu_1m_nw4_shufchunks.log @@ -0,0 +1,4254 @@ + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[rank4]: Traceback (most recent call last): +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank4]: main() +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 302, in main +[rank4]: torch.cuda.set_device(local_rank) +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/cuda/__init__.py", line 477, in set_device +[rank4]: torch._C._cuda_setDevice(device) +[rank4]: RuntimeError: CUDA error: invalid device ordinal +[rank4]: CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. +[rank4]: For debugging consider passing CUDA_LAUNCH_BLOCKING=1 +[rank4]: Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions. + +[rank5]: Traceback (most recent call last): +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank5]: main() +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 302, in main +[rank5]: torch.cuda.set_device(local_rank) +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/cuda/__init__.py", line 477, in set_device +[rank5]: torch._C._cuda_setDevice(device) +[rank5]: RuntimeError: CUDA error: invalid device ordinal +[rank5]: CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. +[rank5]: For debugging consider passing CUDA_LAUNCH_BLOCKING=1 +[rank5]: Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions. + +[rank6]: Traceback (most recent call last): +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank6]: main() +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 302, in main +[rank6]: torch.cuda.set_device(local_rank) +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/cuda/__init__.py", line 477, in set_device +[rank6]: torch._C._cuda_setDevice(device) +[rank6]: RuntimeError: CUDA error: invalid device ordinal +[rank6]: CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. +[rank6]: For debugging consider passing CUDA_LAUNCH_BLOCKING=1 +[rank6]: Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions. + +[rank7]: Traceback (most recent call last): +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank7]: main() +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 302, in main +[rank7]: torch.cuda.set_device(local_rank) +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/cuda/__init__.py", line 477, in set_device +[rank7]: torch._C._cuda_setDevice(device) +[rank7]: RuntimeError: CUDA error: invalid device ordinal +[rank7]: CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. +[rank7]: For debugging consider passing CUDA_LAUNCH_BLOCKING=1 +[rank7]: Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions. + +[rank4]:[W509 21:46:43.430960314 ProcessGroupNCCL.cpp:1487] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W509 21:46:43.606477212 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +W0509 21:46:43.956000 609533 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 609600 closing signal SIGTERM +W0509 21:46:43.956000 609533 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 609601 closing signal SIGTERM +W0509 21:46:43.957000 609533 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 609602 closing signal SIGTERM +W0509 21:46:43.958000 609533 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 609603 closing signal SIGTERM +W0509 21:46:43.958000 609533 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 609605 closing signal SIGTERM +W0509 21:46:43.958000 609533 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 609606 closing signal SIGTERM +W0509 21:46:43.959000 609533 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 609607 closing signal SIGTERM +E0509 21:46:44.087000 609533 torch/distributed/elastic/multiprocessing/api.py:870] failed (exitcode: 1) local_rank: 4 (pid: 609604) of binary: /usr/bin/python +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +train.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-05-09_21:46:43 + host : localhost + rank : 4 (local_rank: 4) + exitcode : 1 (pid: 609604) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[rank6]:[W509 21:56:12.606022394 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank4]:[W509 21:56:12.736712548 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank3]:[W509 21:56:12.766391005 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank1]:[W509 21:56:12.771672089 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank5]:[W509 21:56:12.812641602 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank2]:[W509 21:56:12.813563576 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank7]:[W509 21:56:12.816727456 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank6]:[W509 22:06:12.673705105 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:43562, remote=[localhost]:29672) returned 0, likely a timeout +[rank6]:[W509 22:06:12.673794185 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:43562, remote=[localhost]:29672) timed out after 600000ms +[rank6]: Traceback (most recent call last): +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank6]: main() +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank6]: dist.barrier() +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank6]: return func(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank6]: work = group.barrier(opts=opts) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: torch.distributed.DistBackendError: [6] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank6]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank6]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f3ec412f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank6]: frame #1: + 0x10b6c9e (0x7f3f1404ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f3f186eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f3f186ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f3f1868e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f3f1868e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f3f1868e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f3ec4e6217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f3ec4e6e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #9: + 0xcd3cb3 (0x7f3ec4e73cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f3ec4e744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f3ec4e8413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #12: + 0x56e465f (0x7f3f1867c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #13: + 0x56f3cde (0x7f3f1868bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #14: + 0x4dbe9d0 (0x7f3f17d569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #15: + 0x57029b2 (0x7f3f1869a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #16: + 0x570376d (0x7f3f1869b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #17: + 0xc6875f (0x7f3f208a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank6]: frame #18: + 0x37811d (0x7f3f1ffb411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank6]: frame #19: /usr/bin/python() [0x58208f] +[rank6]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank6]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank6]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank6]: frame #23: /usr/bin/python() [0x608b42] +[rank6]: frame #24: /usr/bin/python() [0x6b4e93] +[rank6]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank6]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank6]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank6]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank6]: frame #29: + 0x2a1ca (0x7f3f2c8571ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank6]: frame #30: __libc_start_main + 0x8b (0x7f3f2c85728b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank6]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank6]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank4]:[W509 22:06:12.741964580 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:43532, remote=[localhost]:29672) returned 0, likely a timeout +[rank4]:[W509 22:06:12.742078235 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:43532, remote=[localhost]:29672) timed out after 600000ms +[rank4]: Traceback (most recent call last): +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank4]: main() +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank4]: dist.barrier() +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank4]: return func(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank4]: work = group.barrier(opts=opts) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: torch.distributed.DistBackendError: [4] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank4]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank4]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f00ebb6c818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank4]: frame #1: + 0x10b6c9e (0x7f013bb33c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f01401cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f01401d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f014017392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f014017392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f014017392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f00ec94717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f00ec9532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #9: + 0xcd3cb3 (0x7f00ec958cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f00ec9594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f00ec96913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #12: + 0x56e465f (0x7f014016165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #13: + 0x56f3cde (0x7f0140170cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #14: + 0x4dbe9d0 (0x7f013f83b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #15: + 0x57029b2 (0x7f014017f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #16: + 0x570376d (0x7f014018076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #17: + 0xc6875f (0x7f014838975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank4]: frame #18: + 0x37811d (0x7f0147a9911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank4]: frame #19: /usr/bin/python() [0x58208f] +[rank4]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank4]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank4]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank4]: frame #23: /usr/bin/python() [0x608b42] +[rank4]: frame #24: /usr/bin/python() [0x6b4e93] +[rank4]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank4]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank4]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank4]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank4]: frame #29: + 0x2a1ca (0x7f01543761ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank4]: frame #30: __libc_start_main + 0x8b (0x7f015437628b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank4]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank4]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank2]:[W509 22:06:12.824237476 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:43548, remote=[localhost]:29672) returned 0, likely a timeout +[rank2]:[W509 22:06:12.824317937 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:43548, remote=[localhost]:29672) timed out after 600000ms +[rank2]: Traceback (most recent call last): +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank2]: main() +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank2]: dist.barrier() +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank2]: return func(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank2]: work = group.barrier(opts=opts) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: torch.distributed.DistBackendError: [2] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank2]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank2]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f34ba12f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank2]: frame #1: + 0x10b6c9e (0x7f350a04ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f350e6eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f350e6ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f350e68e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f350e68e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f350e68e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f34bae6217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f34bae6e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #9: + 0xcd3cb3 (0x7f34bae73cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f34bae744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f34bae8413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #12: + 0x56e465f (0x7f350e67c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #13: + 0x56f3cde (0x7f350e68bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #14: + 0x4dbe9d0 (0x7f350dd569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #15: + 0x57029b2 (0x7f350e69a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #16: + 0x570376d (0x7f350e69b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #17: + 0xc6875f (0x7f35168a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank2]: frame #18: + 0x37811d (0x7f3515fb411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank2]: frame #19: /usr/bin/python() [0x58208f] +[rank2]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank2]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank2]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank2]: frame #23: /usr/bin/python() [0x608b42] +[rank2]: frame #24: /usr/bin/python() [0x6b4e93] +[rank2]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank2]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank2]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank2]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank2]: frame #29: + 0x2a1ca (0x7f352288f1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank2]: frame #30: __libc_start_main + 0x8b (0x7f352288f28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank2]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank2]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank3]:[W509 22:06:12.870025298 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:43578, remote=[localhost]:29672) returned 0, likely a timeout +[rank3]:[W509 22:06:12.870086983 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:43578, remote=[localhost]:29672) timed out after 600000ms +[rank1]:[W509 22:06:12.878398514 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:43608, remote=[localhost]:29672) returned 0, likely a timeout +[rank1]:[W509 22:06:12.878483514 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:43608, remote=[localhost]:29672) timed out after 600000ms +[rank3]: Traceback (most recent call last): +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank3]: main() +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank3]: dist.barrier() +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank3]: return func(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank3]: work = group.barrier(opts=opts) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: torch.distributed.DistBackendError: [3] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank3]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank3]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f2214d6c818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank3]: frame #1: + 0x10b6c9e (0x7f2264d33c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f22693cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f22693d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f226937392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f226937392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f226937392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f2215b4717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f2215b532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #9: + 0xcd3cb3 (0x7f2215b58cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f2215b594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f2215b6913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #12: + 0x56e465f (0x7f226936165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #13: + 0x56f3cde (0x7f2269370cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #14: + 0x4dbe9d0 (0x7f2268a3b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #15: + 0x57029b2 (0x7f226937f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #16: + 0x570376d (0x7f226938076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #17: + 0xc6875f (0x7f227158975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank3]: frame #18: + 0x37811d (0x7f2270c9911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank3]: frame #19: /usr/bin/python() [0x58208f] +[rank3]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank3]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank3]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank3]: frame #23: /usr/bin/python() [0x608b42] +[rank3]: frame #24: /usr/bin/python() [0x6b4e93] +[rank3]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank3]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank3]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank3]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank3]: frame #29: + 0x2a1ca (0x7f227d5741ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank3]: frame #30: __libc_start_main + 0x8b (0x7f227d57428b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank3]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank3]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank1]: Traceback (most recent call last): +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank1]: main() +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank1]: dist.barrier() +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank1]: return func(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank1]: work = group.barrier(opts=opts) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: torch.distributed.DistBackendError: [1] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank1]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank1]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f8f70414818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank1]: frame #1: + 0x10b6c9e (0x7f8fc0333c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f8fc49cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f8fc49d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f8fc497392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f8fc497392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f8fc497392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f8f7114717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f8f711532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #9: + 0xcd3cb3 (0x7f8f71158cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f8f711594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f8f7116913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #12: + 0x56e465f (0x7f8fc496165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #13: + 0x56f3cde (0x7f8fc4970cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #14: + 0x4dbe9d0 (0x7f8fc403b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #15: + 0x57029b2 (0x7f8fc497f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #16: + 0x570376d (0x7f8fc498076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #17: + 0xc6875f (0x7f8fccb8975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank1]: frame #18: + 0x37811d (0x7f8fcc29911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank1]: frame #19: /usr/bin/python() [0x58208f] +[rank1]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank1]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank1]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank1]: frame #23: /usr/bin/python() [0x608b42] +[rank1]: frame #24: /usr/bin/python() [0x6b4e93] +[rank1]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank1]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank1]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank1]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank1]: frame #29: + 0x2a1ca (0x7f8fd8b981ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank1]: frame #30: __libc_start_main + 0x8b (0x7f8fd8b9828b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank1]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank1]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank5]:[W509 22:06:13.917081975 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:43528, remote=[localhost]:29672) returned 0, likely a timeout +[rank5]:[W509 22:06:13.917187569 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:43528, remote=[localhost]:29672) timed out after 600000ms +[rank7]:[W509 22:06:13.921045474 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:43530, remote=[localhost]:29672) returned 0, likely a timeout +[rank7]:[W509 22:06:13.921103165 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:43530, remote=[localhost]:29672) timed out after 600000ms +[rank5]: Traceback (most recent call last): +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank5]: main() +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank5]: dist.barrier() +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank5]: return func(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank5]: work = group.barrier(opts=opts) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: torch.distributed.DistBackendError: [5] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank5]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank5]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f30a632f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank5]: frame #1: + 0x10b6c9e (0x7f30f624ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f30fa8eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f30fa8ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f30fa88e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f30fa88e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f30fa88e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f30a706217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f30a706e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #9: + 0xcd3cb3 (0x7f30a7073cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f30a70744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f30a708413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #12: + 0x56e465f (0x7f30fa87c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #13: + 0x56f3cde (0x7f30fa88bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #14: + 0x4dbe9d0 (0x7f30f9f569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #15: + 0x57029b2 (0x7f30fa89a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #16: + 0x570376d (0x7f30fa89b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #17: + 0xc6875f (0x7f3102aa475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank5]: frame #18: + 0x37811d (0x7f31021b411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank5]: frame #19: /usr/bin/python() [0x58208f] +[rank5]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank5]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank5]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank5]: frame #23: /usr/bin/python() [0x608b42] +[rank5]: frame #24: /usr/bin/python() [0x6b4e93] +[rank5]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank5]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank5]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank5]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank5]: frame #29: + 0x2a1ca (0x7f310ea731ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank5]: frame #30: __libc_start_main + 0x8b (0x7f310ea7328b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank5]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank5]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank7]: Traceback (most recent call last): +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank7]: main() +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank7]: dist.barrier() +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank7]: return func(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank7]: work = group.barrier(opts=opts) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: torch.distributed.DistBackendError: [7] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank7]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank7]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7fda7b36c818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank7]: frame #1: + 0x10b6c9e (0x7fdacb333c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7fdacf9cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7fdacf9d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fdacf97392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fdacf97392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fdacf97392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7fda7c14717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7fda7c1532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #9: + 0xcd3cb3 (0x7fda7c158cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7fda7c1594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7fda7c16913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #12: + 0x56e465f (0x7fdacf96165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #13: + 0x56f3cde (0x7fdacf970cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #14: + 0x4dbe9d0 (0x7fdacf03b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #15: + 0x57029b2 (0x7fdacf97f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #16: + 0x570376d (0x7fdacf98076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #17: + 0xc6875f (0x7fdad7b8975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank7]: frame #18: + 0x37811d (0x7fdad729911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank7]: frame #19: /usr/bin/python() [0x58208f] +[rank7]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank7]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank7]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank7]: frame #23: /usr/bin/python() [0x608b42] +[rank7]: frame #24: /usr/bin/python() [0x6b4e93] +[rank7]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank7]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank7]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank7]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank7]: frame #29: + 0x2a1ca (0x7fdae3b121ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank7]: frame #30: __libc_start_main + 0x8b (0x7fdae3b1228b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank7]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank7]: . This may indicate a possible application crash on rank 0 or a network set up issue. +W0509 22:06:13.385000 10234 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10301 closing signal SIGTERM +W0509 22:06:13.385000 10234 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10302 closing signal SIGTERM +W0509 22:06:13.386000 10234 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10305 closing signal SIGTERM +W0509 22:06:13.386000 10234 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10306 closing signal SIGTERM +W0509 22:06:13.387000 10234 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10308 closing signal SIGTERM +E0509 22:06:13.601000 10234 torch/distributed/elastic/multiprocessing/api.py:870] failed (exitcode: 1) local_rank: 2 (pid: 10303) of binary: /usr/bin/python +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +train.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2026-05-09_22:06:13 + host : t-20260510055531-7m4k7-worker-0.t-20260510055531-7m4k7-worker.mlplatform-customtask.svc.cluster.local + rank : 3 (local_rank: 3) + exitcode : 1 (pid: 10304) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[2]: + time : 2026-05-09_22:06:13 + host : t-20260510055531-7m4k7-worker-0.t-20260510055531-7m4k7-worker.mlplatform-customtask.svc.cluster.local + rank : 6 (local_rank: 6) + exitcode : 1 (pid: 10307) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-05-09_22:06:13 + host : t-20260510055531-7m4k7-worker-0.t-20260510055531-7m4k7-worker.mlplatform-customtask.svc.cluster.local + rank : 2 (local_rank: 2) + exitcode : 1 (pid: 10303) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[rank2]:[W509 22:09:36.823280219 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank1]:[W509 22:09:36.832994550 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank6]:[W509 22:09:36.848243359 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank3]:[W509 22:09:36.853559901 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank7]:[W509 22:09:36.865086836 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank4]:[W509 22:09:36.868062600 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank5]:[W509 22:09:36.882659083 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank1]:[W509 22:19:36.874567955 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:51392, remote=[localhost]:29672) returned 0, likely a timeout +[rank1]:[W509 22:19:36.874653263 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:51392, remote=[localhost]:29672) timed out after 600000ms +[rank1]: Traceback (most recent call last): +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank1]: main() +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank1]: dist.barrier() +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank1]: return func(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank1]: work = group.barrier(opts=opts) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: torch.distributed.DistBackendError: [1] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank1]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank1]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f97a076c818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank1]: frame #1: + 0x10b6c9e (0x7f97f0733c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f97f4dcfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f97f4dd0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f97f4d7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f97f4d7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f97f4d7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f97a154717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f97a15532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #9: + 0xcd3cb3 (0x7f97a1558cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f97a15594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f97a156913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #12: + 0x56e465f (0x7f97f4d6165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #13: + 0x56f3cde (0x7f97f4d70cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #14: + 0x4dbe9d0 (0x7f97f443b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #15: + 0x57029b2 (0x7f97f4d7f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #16: + 0x570376d (0x7f97f4d8076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #17: + 0xc6875f (0x7f97fcf8975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank1]: frame #18: + 0x37811d (0x7f97fc69911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank1]: frame #19: /usr/bin/python() [0x58208f] +[rank1]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank1]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank1]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank1]: frame #23: /usr/bin/python() [0x608b42] +[rank1]: frame #24: /usr/bin/python() [0x6b4e93] +[rank1]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank1]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank1]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank1]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank1]: frame #29: + 0x2a1ca (0x7f9808f7a1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank1]: frame #30: __libc_start_main + 0x8b (0x7f9808f7a28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank1]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank1]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank6]:[W509 22:19:36.889230604 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:51386, remote=[localhost]:29672) returned 0, likely a timeout +[rank6]:[W509 22:19:36.889334668 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:51386, remote=[localhost]:29672) timed out after 600000ms +[rank6]: Traceback (most recent call last): +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank6]: main() +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank6]: dist.barrier() +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank6]: return func(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank6]: work = group.barrier(opts=opts) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: torch.distributed.DistBackendError: [6] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank6]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank6]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f0324814818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank6]: frame #1: + 0x10b6c9e (0x7f0374733c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f0378dcfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f0378dd0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f0378d7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f0378d7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f0378d7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f032554717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f03255532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #9: + 0xcd3cb3 (0x7f0325558cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f03255594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f032556913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #12: + 0x56e465f (0x7f0378d6165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #13: + 0x56f3cde (0x7f0378d70cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #14: + 0x4dbe9d0 (0x7f037843b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #15: + 0x57029b2 (0x7f0378d7f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #16: + 0x570376d (0x7f0378d8076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #17: + 0xc6875f (0x7f0380f8975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank6]: frame #18: + 0x37811d (0x7f038069911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank6]: frame #19: /usr/bin/python() [0x58208f] +[rank6]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank6]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank6]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank6]: frame #23: /usr/bin/python() [0x608b42] +[rank6]: frame #24: /usr/bin/python() [0x6b4e93] +[rank6]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank6]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank6]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank6]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank6]: frame #29: + 0x2a1ca (0x7f038cfbb1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank6]: frame #30: __libc_start_main + 0x8b (0x7f038cfbb28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank6]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank6]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank2]:[W509 22:19:36.927919218 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:51442, remote=[localhost]:29672) returned 0, likely a timeout +[rank2]:[W509 22:19:36.928040530 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:51442, remote=[localhost]:29672) timed out after 600000ms +[rank2]: Traceback (most recent call last): +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank2]: main() +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank2]: dist.barrier() +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank2]: return func(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank2]: work = group.barrier(opts=opts) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: torch.distributed.DistBackendError: [2] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank2]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank2]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f339cb2f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank2]: frame #1: + 0x10b6c9e (0x7f33eca4ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f33f10eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f33f10ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f33f108e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f33f108e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f33f108e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f339d86217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f339d86e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #9: + 0xcd3cb3 (0x7f339d873cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f339d8744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f339d88413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #12: + 0x56e465f (0x7f33f107c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #13: + 0x56f3cde (0x7f33f108bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #14: + 0x4dbe9d0 (0x7f33f07569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #15: + 0x57029b2 (0x7f33f109a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #16: + 0x570376d (0x7f33f109b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #17: + 0xc6875f (0x7f33f92a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank2]: frame #18: + 0x37811d (0x7f33f89b411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank2]: frame #19: /usr/bin/python() [0x58208f] +[rank2]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank2]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank2]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank2]: frame #23: /usr/bin/python() [0x608b42] +[rank2]: frame #24: /usr/bin/python() [0x6b4e93] +[rank2]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank2]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank2]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank2]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank2]: frame #29: + 0x2a1ca (0x7f34052691ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank2]: frame #30: __libc_start_main + 0x8b (0x7f340526928b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank2]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank2]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank5]:[W509 22:19:36.940036389 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:51394, remote=[localhost]:29672) returned 0, likely a timeout +[rank5]:[W509 22:19:36.940104251 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:51394, remote=[localhost]:29672) timed out after 600000ms +[rank5]: Traceback (most recent call last): +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank5]: main() +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank5]: dist.barrier() +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank5]: return func(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank5]: work = group.barrier(opts=opts) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: torch.distributed.DistBackendError: [5] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank5]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank5]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7fb1da32f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank5]: frame #1: + 0x10b6c9e (0x7fb22a24ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7fb22e8eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7fb22e8ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fb22e88e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fb22e88e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fb22e88e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7fb1db06217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7fb1db06e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #9: + 0xcd3cb3 (0x7fb1db073cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7fb1db0744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7fb1db08413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #12: + 0x56e465f (0x7fb22e87c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #13: + 0x56f3cde (0x7fb22e88bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #14: + 0x4dbe9d0 (0x7fb22df569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #15: + 0x57029b2 (0x7fb22e89a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #16: + 0x570376d (0x7fb22e89b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #17: + 0xc6875f (0x7fb236aa475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank5]: frame #18: + 0x37811d (0x7fb2361b411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank5]: frame #19: /usr/bin/python() [0x58208f] +[rank5]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank5]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank5]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank5]: frame #23: /usr/bin/python() [0x608b42] +[rank5]: frame #24: /usr/bin/python() [0x6b4e93] +[rank5]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank5]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank5]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank5]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank5]: frame #29: + 0x2a1ca (0x7fb242ad71ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank5]: frame #30: __libc_start_main + 0x8b (0x7fb242ad728b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank5]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank5]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank3]:[W509 22:19:36.957980594 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:51444, remote=[localhost]:29672) returned 0, likely a timeout +[rank3]:[W509 22:19:36.958067819 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:51444, remote=[localhost]:29672) timed out after 600000ms +[rank3]: Traceback (most recent call last): +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank3]: main() +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank3]: dist.barrier() +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank3]: return func(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank3]: work = group.barrier(opts=opts) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: torch.distributed.DistBackendError: [3] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank3]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank3]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f80b7b6c818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank3]: frame #1: + 0x10b6c9e (0x7f8107b33c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f810c1cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f810c1d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f810c17392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f810c17392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f810c17392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f80b894717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f80b89532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #9: + 0xcd3cb3 (0x7f80b8958cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f80b89594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f80b896913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #12: + 0x56e465f (0x7f810c16165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #13: + 0x56f3cde (0x7f810c170cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #14: + 0x4dbe9d0 (0x7f810b83b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #15: + 0x57029b2 (0x7f810c17f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #16: + 0x570376d (0x7f810c18076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #17: + 0xc6875f (0x7f811438975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank3]: frame #18: + 0x37811d (0x7f8113a9911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank3]: frame #19: /usr/bin/python() [0x58208f] +[rank3]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank3]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank3]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank3]: frame #23: /usr/bin/python() [0x608b42] +[rank3]: frame #24: /usr/bin/python() [0x6b4e93] +[rank3]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank3]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank3]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank3]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank3]: frame #29: + 0x2a1ca (0x7f812032b1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank3]: frame #30: __libc_start_main + 0x8b (0x7f812032b28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank3]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank3]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank7]:[W509 22:19:36.969439082 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:51390, remote=[localhost]:29672) returned 0, likely a timeout +[rank7]:[W509 22:19:36.969510517 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:51390, remote=[localhost]:29672) timed out after 600000ms +[rank4]:[W509 22:19:36.972042438 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:51438, remote=[localhost]:29672) returned 0, likely a timeout +[rank4]:[W509 22:19:36.972103742 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:51438, remote=[localhost]:29672) timed out after 600000ms +[rank7]: Traceback (most recent call last): +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank7]: main() +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank7]: dist.barrier() +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank7]: return func(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank7]: work = group.barrier(opts=opts) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: torch.distributed.DistBackendError: [7] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank7]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank7]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f008dd6c818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank7]: frame #1: + 0x10b6c9e (0x7f00ddd33c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f00e23cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f00e23d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f00e237392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f00e237392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f00e237392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f008eb4717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f008eb532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #9: + 0xcd3cb3 (0x7f008eb58cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f008eb594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f008eb6913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #12: + 0x56e465f (0x7f00e236165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #13: + 0x56f3cde (0x7f00e2370cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #14: + 0x4dbe9d0 (0x7f00e1a3b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #15: + 0x57029b2 (0x7f00e237f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #16: + 0x570376d (0x7f00e238076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #17: + 0xc6875f (0x7f00ea58975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank7]: frame #18: + 0x37811d (0x7f00e9c9911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank7]: frame #19: /usr/bin/python() [0x58208f] +[rank7]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank7]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank7]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank7]: frame #23: /usr/bin/python() [0x608b42] +[rank7]: frame #24: /usr/bin/python() [0x6b4e93] +[rank7]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank7]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank7]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank7]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank7]: frame #29: + 0x2a1ca (0x7f00f65241ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank7]: frame #30: __libc_start_main + 0x8b (0x7f00f652428b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank7]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank7]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank4]: Traceback (most recent call last): +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank4]: main() +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank4]: dist.barrier() +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank4]: return func(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank4]: work = group.barrier(opts=opts) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: torch.distributed.DistBackendError: [4] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank4]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank4]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f8e2b92f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank4]: frame #1: + 0x10b6c9e (0x7f8e7b84ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f8e7feeae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f8e7feebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f8e7fe8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f8e7fe8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f8e7fe8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f8e2c66217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f8e2c66e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #9: + 0xcd3cb3 (0x7f8e2c673cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f8e2c6744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f8e2c68413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #12: + 0x56e465f (0x7f8e7fe7c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #13: + 0x56f3cde (0x7f8e7fe8bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #14: + 0x4dbe9d0 (0x7f8e7f5569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #15: + 0x57029b2 (0x7f8e7fe9a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #16: + 0x570376d (0x7f8e7fe9b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #17: + 0xc6875f (0x7f8e880a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank4]: frame #18: + 0x37811d (0x7f8e877b411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank4]: frame #19: /usr/bin/python() [0x58208f] +[rank4]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank4]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank4]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank4]: frame #23: /usr/bin/python() [0x608b42] +[rank4]: frame #24: /usr/bin/python() [0x6b4e93] +[rank4]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank4]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank4]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank4]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank4]: frame #29: + 0x2a1ca (0x7f8e940571ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank4]: frame #30: __libc_start_main + 0x8b (0x7f8e9405728b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank4]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank4]: . This may indicate a possible application crash on rank 0 or a network set up issue. +W0509 22:19:37.410000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10349 closing signal SIGTERM +W0509 22:19:37.411000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10350 closing signal SIGTERM +W0509 22:19:37.411000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10351 closing signal SIGTERM +W0509 22:19:37.412000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10353 closing signal SIGTERM +W0509 22:19:37.412000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10355 closing signal SIGTERM +E0509 22:19:37.655000 10282 torch/distributed/elastic/multiprocessing/api.py:870] failed (exitcode: 1) local_rank: 3 (pid: 10352) of binary: /usr/bin/python +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +train.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2026-05-09_22:19:37 + host : t-20260510060630-g7xxg-worker-0.t-20260510060630-g7xxg-worker.mlplatform-customtask.svc.cluster.local + rank : 5 (local_rank: 5) + exitcode : 1 (pid: 10354) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[2]: + time : 2026-05-09_22:19:37 + host : t-20260510060630-g7xxg-worker-0.t-20260510060630-g7xxg-worker.mlplatform-customtask.svc.cluster.local + rank : 7 (local_rank: 7) + exitcode : 1 (pid: 10356) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-05-09_22:19:37 + host : t-20260510060630-g7xxg-worker-0.t-20260510060630-g7xxg-worker.mlplatform-customtask.svc.cluster.local + rank : 3 (local_rank: 3) + exitcode : 1 (pid: 10352) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[rank3]:[W509 22:21:26.643748549 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank6]:[W509 22:21:26.669183009 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank7]:[W509 22:21:26.682703152 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank1]:[W509 22:21:26.702502345 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank5]:[W509 22:21:26.705575337 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank4]:[W509 22:21:26.720676248 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank2]:[W509 22:21:26.737147529 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank6]:[W509 22:31:26.711377575 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:39486, remote=[localhost]:29672) returned 0, likely a timeout +[rank6]:[W509 22:31:26.711460856 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:39486, remote=[localhost]:29672) timed out after 600000ms +[rank6]: Traceback (most recent call last): +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank6]: main() +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank6]: dist.barrier() +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank6]: return func(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank6]: work = group.barrier(opts=opts) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: torch.distributed.DistBackendError: [6] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank6]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank6]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f454a92f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank6]: frame #1: + 0x10b6c9e (0x7f459a84ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f459eeeae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f459eeebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f459ee8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f459ee8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f459ee8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f454b66217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f454b66e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #9: + 0xcd3cb3 (0x7f454b673cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f454b6744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f454b68413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #12: + 0x56e465f (0x7f459ee7c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #13: + 0x56f3cde (0x7f459ee8bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #14: + 0x4dbe9d0 (0x7f459e5569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #15: + 0x57029b2 (0x7f459ee9a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #16: + 0x570376d (0x7f459ee9b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #17: + 0xc6875f (0x7f45a70a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank6]: frame #18: + 0x37811d (0x7f45a67b411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank6]: frame #19: /usr/bin/python() [0x58208f] +[rank6]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank6]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank6]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank6]: frame #23: /usr/bin/python() [0x608b42] +[rank6]: frame #24: /usr/bin/python() [0x6b4e93] +[rank6]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank6]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank6]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank6]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank6]: frame #29: + 0x2a1ca (0x7f45b30621ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank6]: frame #30: __libc_start_main + 0x8b (0x7f45b306228b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank6]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank6]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank3]:[W509 22:31:26.748253273 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:39518, remote=[localhost]:29672) returned 0, likely a timeout +[rank3]:[W509 22:31:26.748334705 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:39518, remote=[localhost]:29672) timed out after 600000ms +[rank4]:[W509 22:31:26.753209983 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:39516, remote=[localhost]:29672) returned 0, likely a timeout +[rank4]:[W509 22:31:26.753269012 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:39516, remote=[localhost]:29672) timed out after 600000ms +[rank3]: Traceback (most recent call last): +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank3]: main() +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank3]: dist.barrier() +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank3]: return func(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank3]: work = group.barrier(opts=opts) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: torch.distributed.DistBackendError: [3] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank3]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank3]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f088212f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank3]: frame #1: + 0x10b6c9e (0x7f08d204ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f08d66eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f08d66ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f08d668e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f08d668e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f08d668e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f0882e6217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f0882e6e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #9: + 0xcd3cb3 (0x7f0882e73cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f0882e744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f0882e8413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #12: + 0x56e465f (0x7f08d667c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #13: + 0x56f3cde (0x7f08d668bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #14: + 0x4dbe9d0 (0x7f08d5d569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #15: + 0x57029b2 (0x7f08d669a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #16: + 0x570376d (0x7f08d669b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #17: + 0xc6875f (0x7f08de8a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank3]: frame #18: + 0x37811d (0x7f08ddfb411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank3]: frame #19: /usr/bin/python() [0x58208f] +[rank3]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank3]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank3]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank3]: frame #23: /usr/bin/python() [0x608b42] +[rank3]: frame #24: /usr/bin/python() [0x6b4e93] +[rank3]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank3]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank3]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank3]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank3]: frame #29: + 0x2a1ca (0x7f08ea8da1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank3]: frame #30: __libc_start_main + 0x8b (0x7f08ea8da28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank3]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank3]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank4]: Traceback (most recent call last): +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank4]: main() +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank4]: dist.barrier() +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank4]: return func(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank4]: work = group.barrier(opts=opts) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: torch.distributed.DistBackendError: [4] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank4]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank4]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7fd8d756c818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank4]: frame #1: + 0x10b6c9e (0x7fd927533c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7fd92bbcfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7fd92bbd0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fd92bb7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fd92bb7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fd92bb7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7fd8d834717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7fd8d83532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #9: + 0xcd3cb3 (0x7fd8d8358cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7fd8d83594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7fd8d836913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #12: + 0x56e465f (0x7fd92bb6165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #13: + 0x56f3cde (0x7fd92bb70cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #14: + 0x4dbe9d0 (0x7fd92b23b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #15: + 0x57029b2 (0x7fd92bb7f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #16: + 0x570376d (0x7fd92bb8076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #17: + 0xc6875f (0x7fd933d8975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank4]: frame #18: + 0x37811d (0x7fd93349911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank4]: frame #19: /usr/bin/python() [0x58208f] +[rank4]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank4]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank4]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank4]: frame #23: /usr/bin/python() [0x608b42] +[rank4]: frame #24: /usr/bin/python() [0x6b4e93] +[rank4]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank4]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank4]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank4]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank4]: frame #29: + 0x2a1ca (0x7fd93fd5a1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank4]: frame #30: __libc_start_main + 0x8b (0x7fd93fd5a28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank4]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank4]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank7]:[W509 22:31:26.787071216 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:39528, remote=[localhost]:29672) returned 0, likely a timeout +[rank7]:[W509 22:31:26.787135219 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:39528, remote=[localhost]:29672) timed out after 600000ms +[rank7]: Traceback (most recent call last): +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank7]: main() +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank7]: dist.barrier() +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank7]: return func(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank7]: work = group.barrier(opts=opts) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: torch.distributed.DistBackendError: [7] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank7]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank7]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f0e2872f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank7]: frame #1: + 0x10b6c9e (0x7f0e7864ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f0e7cceae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f0e7ccebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f0e7cc8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f0e7cc8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f0e7cc8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f0e2946217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f0e2946e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #9: + 0xcd3cb3 (0x7f0e29473cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f0e294744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f0e2948413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #12: + 0x56e465f (0x7f0e7cc7c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #13: + 0x56f3cde (0x7f0e7cc8bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #14: + 0x4dbe9d0 (0x7f0e7c3569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #15: + 0x57029b2 (0x7f0e7cc9a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #16: + 0x570376d (0x7f0e7cc9b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #17: + 0xc6875f (0x7f0e84ea475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank7]: frame #18: + 0x37811d (0x7f0e845b411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank7]: frame #19: /usr/bin/python() [0x58208f] +[rank7]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank7]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank7]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank7]: frame #23: /usr/bin/python() [0x608b42] +[rank7]: frame #24: /usr/bin/python() [0x6b4e93] +[rank7]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank7]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank7]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank7]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank7]: frame #29: + 0x2a1ca (0x7f0e90e6a1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank7]: frame #30: __libc_start_main + 0x8b (0x7f0e90e6a28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank7]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank7]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank1]:[W509 22:31:26.799410876 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:39500, remote=[localhost]:29672) returned 0, likely a timeout +[rank1]:[W509 22:31:26.799475582 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:39500, remote=[localhost]:29672) timed out after 600000ms +[rank5]:[W509 22:31:26.809866967 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:39544, remote=[localhost]:29672) returned 0, likely a timeout +[rank5]:[W509 22:31:26.809917316 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:39544, remote=[localhost]:29672) timed out after 600000ms +[rank1]: Traceback (most recent call last): +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank1]: main() +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank1]: dist.barrier() +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank1]: return func(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank1]: work = group.barrier(opts=opts) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: torch.distributed.DistBackendError: [1] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank1]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank1]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f388c16c818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank1]: frame #1: + 0x10b6c9e (0x7f38dc133c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f38e07cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f38e07d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f38e077392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f38e077392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f38e077392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f388cf4717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f388cf532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #9: + 0xcd3cb3 (0x7f388cf58cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f388cf594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f388cf6913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #12: + 0x56e465f (0x7f38e076165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #13: + 0x56f3cde (0x7f38e0770cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #14: + 0x4dbe9d0 (0x7f38dfe3b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #15: + 0x57029b2 (0x7f38e077f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #16: + 0x570376d (0x7f38e078076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #17: + 0xc6875f (0x7f38e898975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank1]: frame #18: + 0x37811d (0x7f38e809911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank1]: frame #19: /usr/bin/python() [0x58208f] +[rank1]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank1]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank1]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank1]: frame #23: /usr/bin/python() [0x608b42] +[rank1]: frame #24: /usr/bin/python() [0x6b4e93] +[rank1]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank1]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank1]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank1]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank1]: frame #29: + 0x2a1ca (0x7f38f48ff1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank1]: frame #30: __libc_start_main + 0x8b (0x7f38f48ff28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank1]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank1]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank5]: Traceback (most recent call last): +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank5]: main() +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank5]: dist.barrier() +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank5]: return func(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank5]: work = group.barrier(opts=opts) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: torch.distributed.DistBackendError: [5] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank5]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank5]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f794672f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank5]: frame #1: + 0x10b6c9e (0x7f799664ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f799aceae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f799acebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f799ac8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f799ac8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f799ac8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f794746217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f794746e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #9: + 0xcd3cb3 (0x7f7947473cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f79474744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f794748413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #12: + 0x56e465f (0x7f799ac7c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #13: + 0x56f3cde (0x7f799ac8bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #14: + 0x4dbe9d0 (0x7f799a3569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #15: + 0x57029b2 (0x7f799ac9a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #16: + 0x570376d (0x7f799ac9b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #17: + 0xc6875f (0x7f79a2ea475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank5]: frame #18: + 0x37811d (0x7f79a25b411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank5]: frame #19: /usr/bin/python() [0x58208f] +[rank5]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank5]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank5]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank5]: frame #23: /usr/bin/python() [0x608b42] +[rank5]: frame #24: /usr/bin/python() [0x6b4e93] +[rank5]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank5]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank5]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank5]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank5]: frame #29: + 0x2a1ca (0x7f79aeea91ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank5]: frame #30: __libc_start_main + 0x8b (0x7f79aeea928b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank5]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank5]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank2]:[W509 22:31:26.841596419 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:39548, remote=[localhost]:29672) returned 0, likely a timeout +[rank2]:[W509 22:31:26.841682178 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:39548, remote=[localhost]:29672) timed out after 600000ms +[rank2]: Traceback (most recent call last): +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank2]: main() +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank2]: dist.barrier() +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank2]: return func(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank2]: work = group.barrier(opts=opts) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: torch.distributed.DistBackendError: [2] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank2]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank2]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f3012e14818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank2]: frame #1: + 0x10b6c9e (0x7f3062d33c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f30673cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f30673d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f306737392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f306737392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f306737392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f3013b4717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f3013b532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #9: + 0xcd3cb3 (0x7f3013b58cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f3013b594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f3013b6913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #12: + 0x56e465f (0x7f306736165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #13: + 0x56f3cde (0x7f3067370cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #14: + 0x4dbe9d0 (0x7f3066a3b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #15: + 0x57029b2 (0x7f306737f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #16: + 0x570376d (0x7f306738076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #17: + 0xc6875f (0x7f306f58975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank2]: frame #18: + 0x37811d (0x7f306ec9911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank2]: frame #19: /usr/bin/python() [0x58208f] +[rank2]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank2]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank2]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank2]: frame #23: /usr/bin/python() [0x608b42] +[rank2]: frame #24: /usr/bin/python() [0x6b4e93] +[rank2]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank2]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank2]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank2]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank2]: frame #29: + 0x2a1ca (0x7f307b5eb1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank2]: frame #30: __libc_start_main + 0x8b (0x7f307b5eb28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank2]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank2]: . This may indicate a possible application crash on rank 0 or a network set up issue. +W0509 22:31:27.079000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10349 closing signal SIGTERM +W0509 22:31:27.080000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10350 closing signal SIGTERM +W0509 22:31:27.080000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10351 closing signal SIGTERM +W0509 22:31:27.081000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10352 closing signal SIGTERM +W0509 22:31:27.081000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10354 closing signal SIGTERM +W0509 22:31:27.082000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10356 closing signal SIGTERM +E0509 22:31:27.410000 10282 torch/distributed/elastic/multiprocessing/api.py:870] failed (exitcode: 1) local_rank: 4 (pid: 10353) of binary: /usr/bin/python +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +train.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2026-05-09_22:31:27 + host : t-20260510061954-q29dv-worker-0.t-20260510061954-q29dv-worker.mlplatform-customtask.svc.cluster.local + rank : 6 (local_rank: 6) + exitcode : 1 (pid: 10355) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-05-09_22:31:27 + host : t-20260510061954-q29dv-worker-0.t-20260510061954-q29dv-worker.mlplatform-customtask.svc.cluster.local + rank : 4 (local_rank: 4) + exitcode : 1 (pid: 10353) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[rank4]:[W509 22:33:15.682191739 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank5]:[W509 22:33:15.737772029 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank7]:[W509 22:33:15.771628308 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank3]:[W509 22:33:15.771931191 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank1]:[W509 22:33:15.776764425 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank2]:[W509 22:33:15.791900849 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank6]:[W509 22:33:15.806219974 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank4]:[W509 22:43:15.791063525 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:36378, remote=[localhost]:29672) returned 0, likely a timeout +[rank4]:[W509 22:43:15.791171093 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:36378, remote=[localhost]:29672) timed out after 600000ms +[rank4]: Traceback (most recent call last): +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank4]: main() +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank4]: dist.barrier() +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank4]: return func(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank4]: work = group.barrier(opts=opts) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: torch.distributed.DistBackendError: [4] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank4]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank4]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7fe8e8287818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank4]: frame #1: + 0x10b6c9e (0x7fe93824ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7fe93c8eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7fe93c8ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fe93c88e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fe93c88e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fe93c88e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7fe8e906217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7fe8e906e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #9: + 0xcd3cb3 (0x7fe8e9073cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7fe8e90744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7fe8e908413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #12: + 0x56e465f (0x7fe93c87c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #13: + 0x56f3cde (0x7fe93c88bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #14: + 0x4dbe9d0 (0x7fe93bf569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #15: + 0x57029b2 (0x7fe93c89a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #16: + 0x570376d (0x7fe93c89b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #17: + 0xc6875f (0x7fe944aa475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank4]: frame #18: + 0x37811d (0x7fe9441b411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank4]: frame #19: /usr/bin/python() [0x58208f] +[rank4]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank4]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank4]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank4]: frame #23: /usr/bin/python() [0x608b42] +[rank4]: frame #24: /usr/bin/python() [0x6b4e93] +[rank4]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank4]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank4]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank4]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank4]: frame #29: + 0x2a1ca (0x7fe950a191ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank4]: frame #30: __libc_start_main + 0x8b (0x7fe950a1928b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank4]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank4]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank3]:[W509 22:43:15.827119862 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:36358, remote=[localhost]:29672) returned 0, likely a timeout +[rank3]:[W509 22:43:15.827202104 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:36358, remote=[localhost]:29672) timed out after 600000ms +[rank3]: Traceback (most recent call last): +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank3]: main() +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank3]: dist.barrier() +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank3]: return func(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank3]: work = group.barrier(opts=opts) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: torch.distributed.DistBackendError: [3] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank3]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank3]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f7d9dc14818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank3]: frame #1: + 0x10b6c9e (0x7f7dedb33c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f7df21cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f7df21d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f7df217392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f7df217392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f7df217392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f7d9e94717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f7d9e9532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #9: + 0xcd3cb3 (0x7f7d9e958cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f7d9e9594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f7d9e96913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #12: + 0x56e465f (0x7f7df216165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #13: + 0x56f3cde (0x7f7df2170cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #14: + 0x4dbe9d0 (0x7f7df183b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #15: + 0x57029b2 (0x7f7df217f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #16: + 0x570376d (0x7f7df218076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #17: + 0xc6875f (0x7f7dfa38975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank3]: frame #18: + 0x37811d (0x7f7df9a9911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank3]: frame #19: /usr/bin/python() [0x58208f] +[rank3]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank3]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank3]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank3]: frame #23: /usr/bin/python() [0x608b42] +[rank3]: frame #24: /usr/bin/python() [0x6b4e93] +[rank3]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank3]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank3]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank3]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank3]: frame #29: + 0x2a1ca (0x7f7e063bd1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank3]: frame #30: __libc_start_main + 0x8b (0x7f7e063bd28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank3]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank3]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank5]:[W509 22:43:15.842373004 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:36398, remote=[localhost]:29672) returned 0, likely a timeout +[rank5]:[W509 22:43:15.842434229 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:36398, remote=[localhost]:29672) timed out after 600000ms +[rank5]: Traceback (most recent call last): +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank5]: main() +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank5]: dist.barrier() +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank5]: return func(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank5]: work = group.barrier(opts=opts) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: torch.distributed.DistBackendError: [5] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank5]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank5]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f66dfa87818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank5]: frame #1: + 0x10b6c9e (0x7f672fa4ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f67340eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f67340ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f673408e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f673408e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f673408e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f66e086217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f66e086e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #9: + 0xcd3cb3 (0x7f66e0873cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f66e08744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f66e088413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #12: + 0x56e465f (0x7f673407c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #13: + 0x56f3cde (0x7f673408bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #14: + 0x4dbe9d0 (0x7f67337569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #15: + 0x57029b2 (0x7f673409a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #16: + 0x570376d (0x7f673409b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #17: + 0xc6875f (0x7f673c2a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank5]: frame #18: + 0x37811d (0x7f673b9b411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank5]: frame #19: /usr/bin/python() [0x58208f] +[rank5]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank5]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank5]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank5]: frame #23: /usr/bin/python() [0x608b42] +[rank5]: frame #24: /usr/bin/python() [0x6b4e93] +[rank5]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank5]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank5]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank5]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank5]: frame #29: + 0x2a1ca (0x7f67482361ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank5]: frame #30: __libc_start_main + 0x8b (0x7f674823628b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank5]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank5]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank2]:[W509 22:43:15.873172481 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:36404, remote=[localhost]:29672) returned 0, likely a timeout +[rank2]:[W509 22:43:15.873242579 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:36404, remote=[localhost]:29672) timed out after 600000ms +[rank7]:[W509 22:43:15.876042725 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:36370, remote=[localhost]:29672) returned 0, likely a timeout +[rank7]:[W509 22:43:15.876126983 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:36370, remote=[localhost]:29672) timed out after 600000ms +[rank1]:[W509 22:43:15.881040390 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:36380, remote=[localhost]:29672) returned 0, likely a timeout +[rank1]:[W509 22:43:15.881118053 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:36380, remote=[localhost]:29672) timed out after 600000ms +[rank2]: Traceback (most recent call last): +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank2]: main() +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank2]: dist.barrier() +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank2]: return func(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank2]: work = group.barrier(opts=opts) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: torch.distributed.DistBackendError: [2] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank2]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank2]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f74b112f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank2]: frame #1: + 0x10b6c9e (0x7f750104ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f75056eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f75056ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f750568e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f750568e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f750568e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f74b1e6217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f74b1e6e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #9: + 0xcd3cb3 (0x7f74b1e73cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f74b1e744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f74b1e8413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #12: + 0x56e465f (0x7f750567c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #13: + 0x56f3cde (0x7f750568bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #14: + 0x4dbe9d0 (0x7f7504d569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #15: + 0x57029b2 (0x7f750569a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #16: + 0x570376d (0x7f750569b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #17: + 0xc6875f (0x7f750d8a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank2]: frame #18: + 0x37811d (0x7f750cfb411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank2]: frame #19: /usr/bin/python() [0x58208f] +[rank2]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank2]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank2]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank2]: frame #23: /usr/bin/python() [0x608b42] +[rank2]: frame #24: /usr/bin/python() [0x6b4e93] +[rank2]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank2]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank2]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank2]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank2]: frame #29: + 0x2a1ca (0x7f75198671ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank2]: frame #30: __libc_start_main + 0x8b (0x7f751986728b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank2]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank2]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank7]: Traceback (most recent call last): +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank7]: main() +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank7]: dist.barrier() +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank7]: return func(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank7]: work = group.barrier(opts=opts) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: torch.distributed.DistBackendError: [7] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank7]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank7]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7fc97fc87818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank7]: frame #1: + 0x10b6c9e (0x7fc9cfc4ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7fc9d42eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7fc9d42ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fc9d428e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fc9d428e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fc9d428e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7fc980a6217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7fc980a6e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #9: + 0xcd3cb3 (0x7fc980a73cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7fc980a744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7fc980a8413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #12: + 0x56e465f (0x7fc9d427c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #13: + 0x56f3cde (0x7fc9d428bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #14: + 0x4dbe9d0 (0x7fc9d39569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #15: + 0x57029b2 (0x7fc9d429a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #16: + 0x570376d (0x7fc9d429b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #17: + 0xc6875f (0x7fc9dc4a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank7]: frame #18: + 0x37811d (0x7fc9dbbb411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank7]: frame #19: /usr/bin/python() [0x58208f] +[rank7]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank7]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank7]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank7]: frame #23: /usr/bin/python() [0x608b42] +[rank7]: frame #24: /usr/bin/python() [0x6b4e93] +[rank7]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank7]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank7]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank7]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank7]: frame #29: + 0x2a1ca (0x7fc9e842b1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank7]: frame #30: __libc_start_main + 0x8b (0x7fc9e842b28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank7]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank7]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank1]: Traceback (most recent call last): +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank1]: main() +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank1]: dist.barrier() +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank1]: return func(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank1]: work = group.barrier(opts=opts) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: torch.distributed.DistBackendError: [1] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank1]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank1]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7ffa5b96c818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank1]: frame #1: + 0x10b6c9e (0x7ffaab933c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7ffaaffcfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7ffaaffd0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7ffaaff7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7ffaaff7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7ffaaff7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7ffa5c74717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7ffa5c7532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #9: + 0xcd3cb3 (0x7ffa5c758cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7ffa5c7594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7ffa5c76913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #12: + 0x56e465f (0x7ffaaff6165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #13: + 0x56f3cde (0x7ffaaff70cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #14: + 0x4dbe9d0 (0x7ffaaf63b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #15: + 0x57029b2 (0x7ffaaff7f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #16: + 0x570376d (0x7ffaaff8076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #17: + 0xc6875f (0x7ffab818975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank1]: frame #18: + 0x37811d (0x7ffab789911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank1]: frame #19: /usr/bin/python() [0x58208f] +[rank1]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank1]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank1]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank1]: frame #23: /usr/bin/python() [0x608b42] +[rank1]: frame #24: /usr/bin/python() [0x6b4e93] +[rank1]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank1]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank1]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank1]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank1]: frame #29: + 0x2a1ca (0x7ffac411e1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank1]: frame #30: __libc_start_main + 0x8b (0x7ffac411e28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank1]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank1]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank6]:[W509 22:43:15.901961274 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:36414, remote=[localhost]:29672) returned 0, likely a timeout +[rank6]:[W509 22:43:15.902030796 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:36414, remote=[localhost]:29672) timed out after 600000ms +[rank6]: Traceback (most recent call last): +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank6]: main() +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank6]: dist.barrier() +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank6]: return func(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank6]: work = group.barrier(opts=opts) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: torch.distributed.DistBackendError: [6] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank6]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank6]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7feb2a087818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank6]: frame #1: + 0x10b6c9e (0x7feb7a04ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7feb7e6eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7feb7e6ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7feb7e68e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7feb7e68e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7feb7e68e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7feb2ae6217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7feb2ae6e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #9: + 0xcd3cb3 (0x7feb2ae73cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7feb2ae744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7feb2ae8413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #12: + 0x56e465f (0x7feb7e67c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #13: + 0x56f3cde (0x7feb7e68bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #14: + 0x4dbe9d0 (0x7feb7dd569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #15: + 0x57029b2 (0x7feb7e69a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #16: + 0x570376d (0x7feb7e69b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #17: + 0xc6875f (0x7feb868a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank6]: frame #18: + 0x37811d (0x7feb85fb411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank6]: frame #19: /usr/bin/python() [0x58208f] +[rank6]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank6]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank6]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank6]: frame #23: /usr/bin/python() [0x608b42] +[rank6]: frame #24: /usr/bin/python() [0x6b4e93] +[rank6]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank6]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank6]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank6]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank6]: frame #29: + 0x2a1ca (0x7feb928401ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank6]: frame #30: __libc_start_main + 0x8b (0x7feb9284028b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank6]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank6]: . This may indicate a possible application crash on rank 0 or a network set up issue. +W0509 22:43:16.313000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10349 closing signal SIGTERM +W0509 22:43:16.314000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10352 closing signal SIGTERM +W0509 22:43:16.315000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10353 closing signal SIGTERM +W0509 22:43:16.315000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10354 closing signal SIGTERM +W0509 22:43:16.315000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10355 closing signal SIGTERM +W0509 22:43:16.316000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10356 closing signal SIGTERM +E0509 22:43:16.581000 10282 torch/distributed/elastic/multiprocessing/api.py:870] failed (exitcode: 1) local_rank: 1 (pid: 10350) of binary: /usr/bin/python +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +train.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2026-05-09_22:43:16 + host : t-20260510063144-ppzxk-worker-0.t-20260510063144-ppzxk-worker.mlplatform-customtask.svc.cluster.local + rank : 2 (local_rank: 2) + exitcode : 1 (pid: 10351) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-05-09_22:43:16 + host : t-20260510063144-ppzxk-worker-0.t-20260510063144-ppzxk-worker.mlplatform-customtask.svc.cluster.local + rank : 1 (local_rank: 1) + exitcode : 1 (pid: 10350) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[rank5]:[W509 22:49:32.031928259 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank4]:[W509 22:49:32.072989078 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank2]:[W509 22:49:32.102232516 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank7]:[W509 22:49:32.159847098 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank3]:[W509 22:49:32.160852602 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank1]:[W509 22:49:32.167053027 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank6]:[W509 22:49:32.182654086 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank5]:[W509 22:59:32.159051423 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:52486, remote=[localhost]:29672) returned 0, likely a timeout +[rank5]:[W509 22:59:32.159158891 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:52486, remote=[localhost]:29672) timed out after 600000ms +[rank3]:[W509 22:59:32.167043423 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:52450, remote=[localhost]:29672) returned 0, likely a timeout +[rank3]:[W509 22:59:32.167141853 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:52450, remote=[localhost]:29672) timed out after 600000ms +[rank5]: Traceback (most recent call last): +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank5]: main() +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank5]: dist.barrier() +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank5]: return func(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank5]: work = group.barrier(opts=opts) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: torch.distributed.DistBackendError: [5] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank5]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank5]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7faf02e14818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank5]: frame #1: + 0x10b6c9e (0x7faf52d33c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7faf573cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7faf573d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7faf5737392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7faf5737392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7faf5737392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7faf03b4717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7faf03b532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #9: + 0xcd3cb3 (0x7faf03b58cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7faf03b594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7faf03b6913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #12: + 0x56e465f (0x7faf5736165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #13: + 0x56f3cde (0x7faf57370cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #14: + 0x4dbe9d0 (0x7faf56a3b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #15: + 0x57029b2 (0x7faf5737f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #16: + 0x570376d (0x7faf5738076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #17: + 0xc6875f (0x7faf5f58975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank5]: frame #18: + 0x37811d (0x7faf5ec9911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank5]: frame #19: /usr/bin/python() [0x58208f] +[rank5]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank5]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank5]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank5]: frame #23: /usr/bin/python() [0x608b42] +[rank5]: frame #24: /usr/bin/python() [0x6b4e93] +[rank5]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank5]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank5]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank5]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank5]: frame #29: + 0x2a1ca (0x7faf6b6091ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank5]: frame #30: __libc_start_main + 0x8b (0x7faf6b60928b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank5]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank5]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank3]: Traceback (most recent call last): +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank3]: main() +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank3]: dist.barrier() +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank3]: return func(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank3]: work = group.barrier(opts=opts) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: torch.distributed.DistBackendError: [3] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank3]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank3]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f3d3f414818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank3]: frame #1: + 0x10b6c9e (0x7f3d8f333c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f3d939cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f3d939d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f3d9397392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f3d9397392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f3d9397392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f3d4014717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f3d401532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #9: + 0xcd3cb3 (0x7f3d40158cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f3d401594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f3d4016913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #12: + 0x56e465f (0x7f3d9396165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #13: + 0x56f3cde (0x7f3d93970cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #14: + 0x4dbe9d0 (0x7f3d9303b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #15: + 0x57029b2 (0x7f3d9397f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #16: + 0x570376d (0x7f3d9398076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #17: + 0xc6875f (0x7f3d9bb8975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank3]: frame #18: + 0x37811d (0x7f3d9b29911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank3]: frame #19: /usr/bin/python() [0x58208f] +[rank3]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank3]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank3]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank3]: frame #23: /usr/bin/python() [0x608b42] +[rank3]: frame #24: /usr/bin/python() [0x6b4e93] +[rank3]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank3]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank3]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank3]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank3]: frame #29: + 0x2a1ca (0x7f3da7bf41ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank3]: frame #30: __libc_start_main + 0x8b (0x7f3da7bf428b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank3]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank3]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank4]:[W509 22:59:32.177683478 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:52478, remote=[localhost]:29672) returned 0, likely a timeout +[rank4]:[W509 22:59:32.177749467 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:52478, remote=[localhost]:29672) timed out after 600000ms +[rank4]: Traceback (most recent call last): +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank4]: main() +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank4]: dist.barrier() +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank4]: return func(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank4]: work = group.barrier(opts=opts) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: torch.distributed.DistBackendError: [4] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank4]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank4]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7fb7b9087818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank4]: frame #1: + 0x10b6c9e (0x7fb80904ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7fb80d6eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7fb80d6ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fb80d68e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fb80d68e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fb80d68e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7fb7b9e6217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7fb7b9e6e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #9: + 0xcd3cb3 (0x7fb7b9e73cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7fb7b9e744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7fb7b9e8413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #12: + 0x56e465f (0x7fb80d67c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #13: + 0x56f3cde (0x7fb80d68bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #14: + 0x4dbe9d0 (0x7fb80cd569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #15: + 0x57029b2 (0x7fb80d69a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #16: + 0x570376d (0x7fb80d69b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #17: + 0xc6875f (0x7fb8158a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank4]: frame #18: + 0x37811d (0x7fb814fb411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank4]: frame #19: /usr/bin/python() [0x58208f] +[rank4]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank4]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank4]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank4]: frame #23: /usr/bin/python() [0x608b42] +[rank4]: frame #24: /usr/bin/python() [0x6b4e93] +[rank4]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank4]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank4]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank4]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank4]: frame #29: + 0x2a1ca (0x7fb82181e1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank4]: frame #30: __libc_start_main + 0x8b (0x7fb82181e28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank4]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank4]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank6]:[W509 22:59:32.200049423 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:52498, remote=[localhost]:29672) returned 0, likely a timeout +[rank6]:[W509 22:59:32.200132490 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:52498, remote=[localhost]:29672) timed out after 600000ms +[rank2]:[W509 22:59:32.207881864 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:52468, remote=[localhost]:29672) returned 0, likely a timeout +[rank2]:[W509 22:59:32.207936220 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:52468, remote=[localhost]:29672) timed out after 600000ms +[rank6]: Traceback (most recent call last): +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank6]: main() +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank6]: dist.barrier() +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank6]: return func(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank6]: work = group.barrier(opts=opts) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: torch.distributed.DistBackendError: [6] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank6]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank6]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f729776c818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank6]: frame #1: + 0x10b6c9e (0x7f72e7733c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f72ebdcfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f72ebdd0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f72ebd7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f72ebd7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f72ebd7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f729854717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f72985532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #9: + 0xcd3cb3 (0x7f7298558cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f72985594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f729856913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #12: + 0x56e465f (0x7f72ebd6165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #13: + 0x56f3cde (0x7f72ebd70cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #14: + 0x4dbe9d0 (0x7f72eb43b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #15: + 0x57029b2 (0x7f72ebd7f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #16: + 0x570376d (0x7f72ebd8076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #17: + 0xc6875f (0x7f72f3f8975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank6]: frame #18: + 0x37811d (0x7f72f369911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank6]: frame #19: /usr/bin/python() [0x58208f] +[rank6]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank6]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank6]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank6]: frame #23: /usr/bin/python() [0x608b42] +[rank6]: frame #24: /usr/bin/python() [0x6b4e93] +[rank6]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank6]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank6]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank6]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank6]: frame #29: + 0x2a1ca (0x7f72fff071ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank6]: frame #30: __libc_start_main + 0x8b (0x7f72fff0728b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank6]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank6]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank2]: Traceback (most recent call last): +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank2]: main() +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank2]: dist.barrier() +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank2]: return func(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank2]: work = group.barrier(opts=opts) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: torch.distributed.DistBackendError: [2] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank2]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank2]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f298b12f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank2]: frame #1: + 0x10b6c9e (0x7f29db04ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f29df6eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f29df6ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f29df68e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f29df68e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f29df68e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f298be6217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f298be6e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #9: + 0xcd3cb3 (0x7f298be73cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f298be744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f298be8413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #12: + 0x56e465f (0x7f29df67c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #13: + 0x56f3cde (0x7f29df68bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #14: + 0x4dbe9d0 (0x7f29ded569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #15: + 0x57029b2 (0x7f29df69a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #16: + 0x570376d (0x7f29df69b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #17: + 0xc6875f (0x7f29e78a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank2]: frame #18: + 0x37811d (0x7f29e6fb411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank2]: frame #19: /usr/bin/python() [0x58208f] +[rank2]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank2]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank2]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank2]: frame #23: /usr/bin/python() [0x608b42] +[rank2]: frame #24: /usr/bin/python() [0x6b4e93] +[rank2]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank2]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank2]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank2]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank2]: frame #29: + 0x2a1ca (0x7f29f38751ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank2]: frame #30: __libc_start_main + 0x8b (0x7f29f387528b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank2]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank2]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank7]:[W509 22:59:32.264311553 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:52462, remote=[localhost]:29672) returned 0, likely a timeout +[rank7]:[W509 22:59:32.264373243 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:52462, remote=[localhost]:29672) timed out after 600000ms +[rank1]:[W509 22:59:32.271502571 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:52514, remote=[localhost]:29672) returned 0, likely a timeout +[rank1]:[W509 22:59:32.271564378 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:52514, remote=[localhost]:29672) timed out after 600000ms +[rank7]: Traceback (most recent call last): +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank7]: main() +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank7]: dist.barrier() +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank7]: return func(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank7]: work = group.barrier(opts=opts) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: torch.distributed.DistBackendError: [7] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank7]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank7]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7ff109414818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank7]: frame #1: + 0x10b6c9e (0x7ff159333c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7ff15d9cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7ff15d9d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7ff15d97392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7ff15d97392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7ff15d97392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7ff10a14717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7ff10a1532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #9: + 0xcd3cb3 (0x7ff10a158cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7ff10a1594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7ff10a16913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #12: + 0x56e465f (0x7ff15d96165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #13: + 0x56f3cde (0x7ff15d970cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #14: + 0x4dbe9d0 (0x7ff15d03b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #15: + 0x57029b2 (0x7ff15d97f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #16: + 0x570376d (0x7ff15d98076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #17: + 0xc6875f (0x7ff165b8975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank7]: frame #18: + 0x37811d (0x7ff16529911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank7]: frame #19: /usr/bin/python() [0x58208f] +[rank7]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank7]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank7]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank7]: frame #23: /usr/bin/python() [0x608b42] +[rank7]: frame #24: /usr/bin/python() [0x6b4e93] +[rank7]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank7]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank7]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank7]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank7]: frame #29: + 0x2a1ca (0x7ff171b991ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank7]: frame #30: __libc_start_main + 0x8b (0x7ff171b9928b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank7]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank7]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank1]: Traceback (most recent call last): +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 625, in +[rank1]: main() +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 318, in main +[rank1]: dist.barrier() +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank1]: return func(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank1]: work = group.barrier(opts=opts) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: torch.distributed.DistBackendError: [1] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank1]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank1]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7eff4476c818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank1]: frame #1: + 0x10b6c9e (0x7eff94733c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7eff98dcfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7eff98dd0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7eff98d7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7eff98d7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7eff98d7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7eff4554717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7eff455532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #9: + 0xcd3cb3 (0x7eff45558cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7eff455594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7eff4556913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #12: + 0x56e465f (0x7eff98d6165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #13: + 0x56f3cde (0x7eff98d70cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #14: + 0x4dbe9d0 (0x7eff9843b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #15: + 0x57029b2 (0x7eff98d7f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #16: + 0x570376d (0x7eff98d8076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #17: + 0xc6875f (0x7effa0f8975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank1]: frame #18: + 0x37811d (0x7effa069911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank1]: frame #19: /usr/bin/python() [0x58208f] +[rank1]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank1]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank1]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank1]: frame #23: /usr/bin/python() [0x608b42] +[rank1]: frame #24: /usr/bin/python() [0x6b4e93] +[rank1]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank1]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank1]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank1]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank1]: frame #29: + 0x2a1ca (0x7effacf4e1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank1]: frame #30: __libc_start_main + 0x8b (0x7effacf4e28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank1]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank1]: . This may indicate a possible application crash on rank 0 or a network set up issue. +W0509 22:59:32.612000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10349 closing signal SIGTERM +W0509 22:59:32.612000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10350 closing signal SIGTERM +W0509 22:59:32.613000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10351 closing signal SIGTERM +W0509 22:59:32.613000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10356 closing signal SIGTERM +E0509 22:59:32.828000 10282 torch/distributed/elastic/multiprocessing/api.py:870] failed (exitcode: 1) local_rank: 3 (pid: 10352) of binary: /usr/bin/python +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +train.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2026-05-09_22:59:32 + host : t-20260510064332-54zb5-worker-0.t-20260510064332-54zb5-worker.mlplatform-customtask.svc.cluster.local + rank : 4 (local_rank: 4) + exitcode : 1 (pid: 10353) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[2]: + time : 2026-05-09_22:59:32 + host : t-20260510064332-54zb5-worker-0.t-20260510064332-54zb5-worker.mlplatform-customtask.svc.cluster.local + rank : 5 (local_rank: 5) + exitcode : 1 (pid: 10354) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[3]: + time : 2026-05-09_22:59:32 + host : t-20260510064332-54zb5-worker-0.t-20260510064332-54zb5-worker.mlplatform-customtask.svc.cluster.local + rank : 6 (local_rank: 6) + exitcode : 1 (pid: 10355) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-05-09_22:59:32 + host : t-20260510064332-54zb5-worker-0.t-20260510064332-54zb5-worker.mlplatform-customtask.svc.cluster.local + rank : 3 (local_rank: 3) + exitcode : 1 (pid: 10352) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[rank1]:[W509 23:01:19.317933866 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank5]:[W509 23:01:19.511234744 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank4]:[W509 23:01:19.511926481 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank6]:[W509 23:01:19.526786123 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank2]:[W509 23:01:19.531683913 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank7]:[W509 23:01:19.551930180 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank3]:[W509 23:01:19.558073904 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank1]:[W509 23:11:19.439409994 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:34500, remote=[localhost]:29672) returned 0, likely a timeout +[rank1]:[W509 23:11:19.439510868 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:34500, remote=[localhost]:29672) timed out after 600000ms +[rank1]: Traceback (most recent call last): +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank1]: main() +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank1]: dist.barrier() +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank1]: return func(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank1]: work = group.barrier(opts=opts) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: torch.distributed.DistBackendError: [1] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank1]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank1]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7fcc0ce14818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank1]: frame #1: + 0x10b6c9e (0x7fcc5cd33c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7fcc613cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7fcc613d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fcc6137392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fcc6137392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fcc6137392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7fcc0db4717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7fcc0db532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #9: + 0xcd3cb3 (0x7fcc0db58cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7fcc0db594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7fcc0db6913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #12: + 0x56e465f (0x7fcc6136165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #13: + 0x56f3cde (0x7fcc61370cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #14: + 0x4dbe9d0 (0x7fcc60a3b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #15: + 0x57029b2 (0x7fcc6137f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #16: + 0x570376d (0x7fcc6138076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #17: + 0xc6875f (0x7fcc6958975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank1]: frame #18: + 0x37811d (0x7fcc68c9911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank1]: frame #19: /usr/bin/python() [0x58208f] +[rank1]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank1]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank1]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank1]: frame #23: /usr/bin/python() [0x608b42] +[rank1]: frame #24: /usr/bin/python() [0x6b4e93] +[rank1]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank1]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank1]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank1]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank1]: frame #29: + 0x2a1ca (0x7fcc755be1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank1]: frame #30: __libc_start_main + 0x8b (0x7fcc755be28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank1]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank1]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank6]:[W509 23:11:19.532086059 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:34508, remote=[localhost]:29672) returned 0, likely a timeout +[rank6]:[W509 23:11:19.532183142 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:34508, remote=[localhost]:29672) timed out after 600000ms +[rank6]: Traceback (most recent call last): +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank6]: main() +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank6]: dist.barrier() +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank6]: return func(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank6]: work = group.barrier(opts=opts) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: torch.distributed.DistBackendError: [6] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank6]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank6]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f9c4d16c818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank6]: frame #1: + 0x10b6c9e (0x7f9c9d133c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f9ca17cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f9ca17d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f9ca177392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f9ca177392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f9ca177392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f9c4df4717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f9c4df532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #9: + 0xcd3cb3 (0x7f9c4df58cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f9c4df594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f9c4df6913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #12: + 0x56e465f (0x7f9ca176165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #13: + 0x56f3cde (0x7f9ca1770cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #14: + 0x4dbe9d0 (0x7f9ca0e3b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #15: + 0x57029b2 (0x7f9ca177f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #16: + 0x570376d (0x7f9ca178076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #17: + 0xc6875f (0x7f9ca998975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank6]: frame #18: + 0x37811d (0x7f9ca909911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank6]: frame #19: /usr/bin/python() [0x58208f] +[rank6]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank6]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank6]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank6]: frame #23: /usr/bin/python() [0x608b42] +[rank6]: frame #24: /usr/bin/python() [0x6b4e93] +[rank6]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank6]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank6]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank6]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank6]: frame #29: + 0x2a1ca (0x7f9cb59341ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank6]: frame #30: __libc_start_main + 0x8b (0x7f9cb593428b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank6]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank6]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank2]:[W509 23:11:19.589988437 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:34494, remote=[localhost]:29672) returned 0, likely a timeout +[rank2]:[W509 23:11:19.590065830 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:34494, remote=[localhost]:29672) timed out after 600000ms +[rank7]:[W509 23:11:19.593053265 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:34524, remote=[localhost]:29672) returned 0, likely a timeout +[rank7]:[W509 23:11:19.593140370 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:34524, remote=[localhost]:29672) timed out after 600000ms +[rank2]: Traceback (most recent call last): +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank2]: main() +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank2]: dist.barrier() +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank2]: return func(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank2]: work = group.barrier(opts=opts) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: torch.distributed.DistBackendError: [2] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank2]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank2]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7fcfc1d2f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank2]: frame #1: + 0x10b6c9e (0x7fd011c4ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7fd0162eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7fd0162ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fd01628e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fd01628e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fd01628e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7fcfc2a6217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7fcfc2a6e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #9: + 0xcd3cb3 (0x7fcfc2a73cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7fcfc2a744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7fcfc2a8413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #12: + 0x56e465f (0x7fd01627c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #13: + 0x56f3cde (0x7fd01628bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #14: + 0x4dbe9d0 (0x7fd0159569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #15: + 0x57029b2 (0x7fd01629a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #16: + 0x570376d (0x7fd01629b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #17: + 0xc6875f (0x7fd01e4a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank2]: frame #18: + 0x37811d (0x7fd01dbb411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank2]: frame #19: /usr/bin/python() [0x58208f] +[rank2]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank2]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank2]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank2]: frame #23: /usr/bin/python() [0x608b42] +[rank2]: frame #24: /usr/bin/python() [0x6b4e93] +[rank2]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank2]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank2]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank2]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank2]: frame #29: + 0x2a1ca (0x7fd02a4fb1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank2]: frame #30: __libc_start_main + 0x8b (0x7fd02a4fb28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank2]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank2]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank7]: Traceback (most recent call last): +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank7]: main() +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank7]: dist.barrier() +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank7]: return func(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank7]: work = group.barrier(opts=opts) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: torch.distributed.DistBackendError: [7] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank7]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank7]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f12ba014818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank7]: frame #1: + 0x10b6c9e (0x7f1309f33c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f130e5cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f130e5d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f130e57392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f130e57392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f130e57392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f12bad4717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f12bad532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #9: + 0xcd3cb3 (0x7f12bad58cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f12bad594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f12bad6913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #12: + 0x56e465f (0x7f130e56165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #13: + 0x56f3cde (0x7f130e570cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #14: + 0x4dbe9d0 (0x7f130dc3b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #15: + 0x57029b2 (0x7f130e57f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #16: + 0x570376d (0x7f130e58076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #17: + 0xc6875f (0x7f131678975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank7]: frame #18: + 0x37811d (0x7f1315e9911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank7]: frame #19: /usr/bin/python() [0x58208f] +[rank7]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank7]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank7]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank7]: frame #23: /usr/bin/python() [0x608b42] +[rank7]: frame #24: /usr/bin/python() [0x6b4e93] +[rank7]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank7]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank7]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank7]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank7]: frame #29: + 0x2a1ca (0x7f13227e71ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank7]: frame #30: __libc_start_main + 0x8b (0x7f13227e728b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank7]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank7]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank3]:[W509 23:11:19.607248558 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:34542, remote=[localhost]:29672) returned 0, likely a timeout +[rank3]:[W509 23:11:19.607319401 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:34542, remote=[localhost]:29672) timed out after 600000ms +[rank5]:[W509 23:11:19.615817750 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:34480, remote=[localhost]:29672) returned 0, likely a timeout +[rank5]:[W509 23:11:19.615905421 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:34480, remote=[localhost]:29672) timed out after 600000ms +[rank4]:[W509 23:11:19.616471946 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:34526, remote=[localhost]:29672) returned 0, likely a timeout +[rank4]:[W509 23:11:19.616539188 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:34526, remote=[localhost]:29672) timed out after 600000ms +[rank3]: Traceback (most recent call last): +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank3]: main() +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank3]: dist.barrier() +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank3]: return func(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank3]: work = group.barrier(opts=opts) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: torch.distributed.DistBackendError: [3] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank3]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank3]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f60fa087818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank3]: frame #1: + 0x10b6c9e (0x7f614a04ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f614e6eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f614e6ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f614e68e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f614e68e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f614e68e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f60fae6217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f60fae6e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #9: + 0xcd3cb3 (0x7f60fae73cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f60fae744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f60fae8413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #12: + 0x56e465f (0x7f614e67c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #13: + 0x56f3cde (0x7f614e68bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #14: + 0x4dbe9d0 (0x7f614dd569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #15: + 0x57029b2 (0x7f614e69a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #16: + 0x570376d (0x7f614e69b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #17: + 0xc6875f (0x7f61568a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank3]: frame #18: + 0x37811d (0x7f6155fb411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank3]: frame #19: /usr/bin/python() [0x58208f] +[rank3]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank3]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank3]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank3]: frame #23: /usr/bin/python() [0x608b42] +[rank3]: frame #24: /usr/bin/python() [0x6b4e93] +[rank3]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank3]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank3]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank3]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank3]: frame #29: + 0x2a1ca (0x7f616283a1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank3]: frame #30: __libc_start_main + 0x8b (0x7f616283a28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank3]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank3]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank5]: Traceback (most recent call last): +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank5]: main() +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank5]: dist.barrier() +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank5]: return func(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank5]: work = group.barrier(opts=opts) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: torch.distributed.DistBackendError: [5] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank5]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank5]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f13dd32f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank5]: frame #1: + 0x10b6c9e (0x7f142d24ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f14318eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f14318ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f143188e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f143188e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f143188e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f13de06217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f13de06e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #9: + 0xcd3cb3 (0x7f13de073cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f13de0744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f13de08413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #12: + 0x56e465f (0x7f143187c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #13: + 0x56f3cde (0x7f143188bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #14: + 0x4dbe9d0 (0x7f1430f569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #15: + 0x57029b2 (0x7f143189a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #16: + 0x570376d (0x7f143189b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #17: + 0xc6875f (0x7f1439aa475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank5]: frame #18: + 0x37811d (0x7f14391b411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank5]: frame #19: /usr/bin/python() [0x58208f] +[rank5]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank5]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank5]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank5]: frame #23: /usr/bin/python() [0x608b42] +[rank5]: frame #24: /usr/bin/python() [0x6b4e93] +[rank5]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank5]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank5]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank5]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank5]: frame #29: + 0x2a1ca (0x7f1445ab91ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank5]: frame #30: __libc_start_main + 0x8b (0x7f1445ab928b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank5]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank5]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank4]: Traceback (most recent call last): +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank4]: main() +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank4]: dist.barrier() +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank4]: return func(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank4]: work = group.barrier(opts=opts) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: torch.distributed.DistBackendError: [4] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank4]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank4]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7fcf66014818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank4]: frame #1: + 0x10b6c9e (0x7fcfb5f33c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7fcfba5cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7fcfba5d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fcfba57392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fcfba57392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fcfba57392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7fcf66d4717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7fcf66d532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #9: + 0xcd3cb3 (0x7fcf66d58cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7fcf66d594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7fcf66d6913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #12: + 0x56e465f (0x7fcfba56165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #13: + 0x56f3cde (0x7fcfba570cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #14: + 0x4dbe9d0 (0x7fcfb9c3b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #15: + 0x57029b2 (0x7fcfba57f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #16: + 0x570376d (0x7fcfba58076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #17: + 0xc6875f (0x7fcfc278975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank4]: frame #18: + 0x37811d (0x7fcfc1e9911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank4]: frame #19: /usr/bin/python() [0x58208f] +[rank4]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank4]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank4]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank4]: frame #23: /usr/bin/python() [0x608b42] +[rank4]: frame #24: /usr/bin/python() [0x6b4e93] +[rank4]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank4]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank4]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank4]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank4]: frame #29: + 0x2a1ca (0x7fcfce7f01ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank4]: frame #30: __libc_start_main + 0x8b (0x7fcfce7f028b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank4]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank4]: . This may indicate a possible application crash on rank 0 or a network set up issue. +W0509 23:11:19.815000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10349 closing signal SIGTERM +W0509 23:11:19.815000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10351 closing signal SIGTERM +W0509 23:11:19.816000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10352 closing signal SIGTERM +W0509 23:11:19.816000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10353 closing signal SIGTERM +W0509 23:11:19.817000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10354 closing signal SIGTERM +W0509 23:11:19.817000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10355 closing signal SIGTERM +W0509 23:11:19.817000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10356 closing signal SIGTERM +E0509 23:11:20.159000 10282 torch/distributed/elastic/multiprocessing/api.py:870] failed (exitcode: 1) local_rank: 1 (pid: 10350) of binary: /usr/bin/python +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +train.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-05-09_23:11:19 + host : t-20260510065949-qt9jg-worker-0.t-20260510065949-qt9jg-worker.mlplatform-customtask.svc.cluster.local + rank : 1 (local_rank: 1) + exitcode : 1 (pid: 10350) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[rank5]:[W509 23:13:10.525550504 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank7]:[W509 23:13:10.559873469 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank1]:[W509 23:13:10.587908110 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank2]:[W509 23:13:10.629755463 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank6]:[W509 23:13:10.702541106 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank4]:[W509 23:13:10.703391651 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank3]:[W509 23:13:10.714969323 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank7]:[W509 23:23:10.622040545 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:57898, remote=[localhost]:29672) returned 0, likely a timeout +[rank7]:[W509 23:23:10.622131904 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:57898, remote=[localhost]:29672) timed out after 600000ms +[rank1]:[W509 23:23:10.630045926 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:57864, remote=[localhost]:29672) returned 0, likely a timeout +[rank1]:[W509 23:23:10.630124821 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:57864, remote=[localhost]:29672) timed out after 600000ms +[rank7]: Traceback (most recent call last): +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank7]: main() +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank7]: dist.barrier() +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank7]: return func(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank7]: work = group.barrier(opts=opts) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: torch.distributed.DistBackendError: [7] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank7]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank7]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f1a7d92f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank7]: frame #1: + 0x10b6c9e (0x7f1acd84ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f1ad1eeae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f1ad1eebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f1ad1e8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f1ad1e8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f1ad1e8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f1a7e66217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f1a7e66e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #9: + 0xcd3cb3 (0x7f1a7e673cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f1a7e6744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f1a7e68413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #12: + 0x56e465f (0x7f1ad1e7c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #13: + 0x56f3cde (0x7f1ad1e8bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #14: + 0x4dbe9d0 (0x7f1ad15569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #15: + 0x57029b2 (0x7f1ad1e9a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #16: + 0x570376d (0x7f1ad1e9b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #17: + 0xc6875f (0x7f1ada0a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank7]: frame #18: + 0x37811d (0x7f1ad97b411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank7]: frame #19: /usr/bin/python() [0x58208f] +[rank7]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank7]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank7]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank7]: frame #23: /usr/bin/python() [0x608b42] +[rank7]: frame #24: /usr/bin/python() [0x6b4e93] +[rank7]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank7]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank7]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank7]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank7]: frame #29: + 0x2a1ca (0x7f1ae60fa1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank7]: frame #30: __libc_start_main + 0x8b (0x7f1ae60fa28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank7]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank7]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank5]:[W509 23:23:10.638040611 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:57840, remote=[localhost]:29672) returned 0, likely a timeout +[rank5]:[W509 23:23:10.638109906 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:57840, remote=[localhost]:29672) timed out after 600000ms +[rank1]: Traceback (most recent call last): +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank1]: main() +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank1]: dist.barrier() +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank1]: return func(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank1]: work = group.barrier(opts=opts) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: torch.distributed.DistBackendError: [1] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank1]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank1]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f239d96c818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank1]: frame #1: + 0x10b6c9e (0x7f23ed933c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f23f1fcfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f23f1fd0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f23f1f7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f23f1f7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f23f1f7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f239e74717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f239e7532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #9: + 0xcd3cb3 (0x7f239e758cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f239e7594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f239e76913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #12: + 0x56e465f (0x7f23f1f6165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #13: + 0x56f3cde (0x7f23f1f70cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #14: + 0x4dbe9d0 (0x7f23f163b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #15: + 0x57029b2 (0x7f23f1f7f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #16: + 0x570376d (0x7f23f1f8076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #17: + 0xc6875f (0x7f23fa18975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank1]: frame #18: + 0x37811d (0x7f23f989911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank1]: frame #19: /usr/bin/python() [0x58208f] +[rank1]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank1]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank1]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank1]: frame #23: /usr/bin/python() [0x608b42] +[rank1]: frame #24: /usr/bin/python() [0x6b4e93] +[rank1]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank1]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank1]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank1]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank1]: frame #29: + 0x2a1ca (0x7f24061321ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank1]: frame #30: __libc_start_main + 0x8b (0x7f240613228b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank1]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank1]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank5]: Traceback (most recent call last): +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank5]: main() +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank5]: dist.barrier() +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank5]: return func(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank5]: work = group.barrier(opts=opts) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: torch.distributed.DistBackendError: [5] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank5]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank5]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f1de656c818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank5]: frame #1: + 0x10b6c9e (0x7f1e36533c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f1e3abcfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f1e3abd0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f1e3ab7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f1e3ab7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f1e3ab7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f1de734717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f1de73532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #9: + 0xcd3cb3 (0x7f1de7358cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f1de73594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f1de736913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #12: + 0x56e465f (0x7f1e3ab6165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #13: + 0x56f3cde (0x7f1e3ab70cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #14: + 0x4dbe9d0 (0x7f1e3a23b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #15: + 0x57029b2 (0x7f1e3ab7f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #16: + 0x570376d (0x7f1e3ab8076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #17: + 0xc6875f (0x7f1e42d8975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank5]: frame #18: + 0x37811d (0x7f1e4249911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank5]: frame #19: /usr/bin/python() [0x58208f] +[rank5]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank5]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank5]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank5]: frame #23: /usr/bin/python() [0x608b42] +[rank5]: frame #24: /usr/bin/python() [0x6b4e93] +[rank5]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank5]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank5]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank5]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank5]: frame #29: + 0x2a1ca (0x7f1e4ed431ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank5]: frame #30: __libc_start_main + 0x8b (0x7f1e4ed4328b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank5]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank5]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank2]:[W509 23:23:10.680159105 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:57838, remote=[localhost]:29672) returned 0, likely a timeout +[rank2]:[W509 23:23:10.680229256 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:57838, remote=[localhost]:29672) timed out after 600000ms +[rank2]: Traceback (most recent call last): +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank2]: main() +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank2]: dist.barrier() +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank2]: return func(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank2]: work = group.barrier(opts=opts) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: torch.distributed.DistBackendError: [2] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank2]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank2]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f4cbe014818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank2]: frame #1: + 0x10b6c9e (0x7f4d0df33c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f4d125cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f4d125d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f4d1257392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f4d1257392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f4d1257392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f4cbed4717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f4cbed532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #9: + 0xcd3cb3 (0x7f4cbed58cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f4cbed594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f4cbed6913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #12: + 0x56e465f (0x7f4d1256165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #13: + 0x56f3cde (0x7f4d12570cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #14: + 0x4dbe9d0 (0x7f4d11c3b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #15: + 0x57029b2 (0x7f4d1257f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #16: + 0x570376d (0x7f4d1258076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #17: + 0xc6875f (0x7f4d1a78975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank2]: frame #18: + 0x37811d (0x7f4d19e9911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank2]: frame #19: /usr/bin/python() [0x58208f] +[rank2]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank2]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank2]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank2]: frame #23: /usr/bin/python() [0x608b42] +[rank2]: frame #24: /usr/bin/python() [0x6b4e93] +[rank2]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank2]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank2]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank2]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank2]: frame #29: + 0x2a1ca (0x7f4d267f11ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank2]: frame #30: __libc_start_main + 0x8b (0x7f4d267f128b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank2]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank2]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank6]:[W509 23:23:10.713514694 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:57878, remote=[localhost]:29672) returned 0, likely a timeout +[rank6]:[W509 23:23:10.713619409 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:57878, remote=[localhost]:29672) timed out after 600000ms +[rank6]: Traceback (most recent call last): +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank6]: main() +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank6]: dist.barrier() +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank6]: return func(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank6]: work = group.barrier(opts=opts) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: torch.distributed.DistBackendError: [6] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank6]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank6]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7ff87f12f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank6]: frame #1: + 0x10b6c9e (0x7ff8cf04ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7ff8d36eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7ff8d36ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7ff8d368e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7ff8d368e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7ff8d368e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7ff87fe6217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7ff87fe6e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #9: + 0xcd3cb3 (0x7ff87fe73cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7ff87fe744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7ff87fe8413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #12: + 0x56e465f (0x7ff8d367c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #13: + 0x56f3cde (0x7ff8d368bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #14: + 0x4dbe9d0 (0x7ff8d2d569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #15: + 0x57029b2 (0x7ff8d369a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #16: + 0x570376d (0x7ff8d369b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #17: + 0xc6875f (0x7ff8db8a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank6]: frame #18: + 0x37811d (0x7ff8dafb411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank6]: frame #19: /usr/bin/python() [0x58208f] +[rank6]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank6]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank6]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank6]: frame #23: /usr/bin/python() [0x608b42] +[rank6]: frame #24: /usr/bin/python() [0x6b4e93] +[rank6]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank6]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank6]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank6]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank6]: frame #29: + 0x2a1ca (0x7ff8e78531ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank6]: frame #30: __libc_start_main + 0x8b (0x7ff8e785328b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank6]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank6]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank4]:[W509 23:23:10.800044394 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:57886, remote=[localhost]:29672) returned 0, likely a timeout +[rank4]:[W509 23:23:10.800148751 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:57886, remote=[localhost]:29672) timed out after 600000ms +[rank4]: Traceback (most recent call last): +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank4]: main() +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank4]: dist.barrier() +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank4]: return func(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank4]: work = group.barrier(opts=opts) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: torch.distributed.DistBackendError: [4] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank4]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank4]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f854be87818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank4]: frame #1: + 0x10b6c9e (0x7f859be4ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f85a04eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f85a04ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f85a048e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f85a048e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f85a048e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f854cc6217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f854cc6e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #9: + 0xcd3cb3 (0x7f854cc73cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f854cc744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f854cc8413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #12: + 0x56e465f (0x7f85a047c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #13: + 0x56f3cde (0x7f85a048bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #14: + 0x4dbe9d0 (0x7f859fb569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #15: + 0x57029b2 (0x7f85a049a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #16: + 0x570376d (0x7f85a049b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #17: + 0xc6875f (0x7f85a86a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank4]: frame #18: + 0x37811d (0x7f85a7db411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank4]: frame #19: /usr/bin/python() [0x58208f] +[rank4]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank4]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank4]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank4]: frame #23: /usr/bin/python() [0x608b42] +[rank4]: frame #24: /usr/bin/python() [0x6b4e93] +[rank4]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank4]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank4]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank4]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank4]: frame #29: + 0x2a1ca (0x7f85b46371ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank4]: frame #30: __libc_start_main + 0x8b (0x7f85b463728b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank4]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank4]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank3]:[W509 23:23:10.819279454 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:57848, remote=[localhost]:29672) returned 0, likely a timeout +[rank3]:[W509 23:23:10.819373401 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:57848, remote=[localhost]:29672) timed out after 600000ms +[rank3]: Traceback (most recent call last): +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank3]: main() +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank3]: dist.barrier() +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank3]: return func(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank3]: work = group.barrier(opts=opts) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: torch.distributed.DistBackendError: [3] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank3]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank3]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f3e1312f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank3]: frame #1: + 0x10b6c9e (0x7f3e6304ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f3e676eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f3e676ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f3e6768e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f3e6768e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f3e6768e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f3e13e6217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f3e13e6e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #9: + 0xcd3cb3 (0x7f3e13e73cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f3e13e744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f3e13e8413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #12: + 0x56e465f (0x7f3e6767c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #13: + 0x56f3cde (0x7f3e6768bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #14: + 0x4dbe9d0 (0x7f3e66d569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #15: + 0x57029b2 (0x7f3e6769a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #16: + 0x570376d (0x7f3e6769b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #17: + 0xc6875f (0x7f3e6f8a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank3]: frame #18: + 0x37811d (0x7f3e6efb411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank3]: frame #19: /usr/bin/python() [0x58208f] +[rank3]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank3]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank3]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank3]: frame #23: /usr/bin/python() [0x608b42] +[rank3]: frame #24: /usr/bin/python() [0x6b4e93] +[rank3]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank3]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank3]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank3]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank3]: frame #29: + 0x2a1ca (0x7f3e7b89c1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank3]: frame #30: __libc_start_main + 0x8b (0x7f3e7b89c28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank3]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank3]: . This may indicate a possible application crash on rank 0 or a network set up issue. +W0509 23:23:10.944000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10349 closing signal SIGTERM +W0509 23:23:10.944000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10350 closing signal SIGTERM +W0509 23:23:10.945000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10351 closing signal SIGTERM +W0509 23:23:10.945000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10352 closing signal SIGTERM +W0509 23:23:10.945000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10353 closing signal SIGTERM +W0509 23:23:10.946000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10355 closing signal SIGTERM +E0509 23:23:11.224000 10282 torch/distributed/elastic/multiprocessing/api.py:870] failed (exitcode: 1) local_rank: 5 (pid: 10354) of binary: /usr/bin/python +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +train.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2026-05-09_23:23:10 + host : t-20260510071136-9xm8l-worker-0.t-20260510071136-9xm8l-worker.mlplatform-customtask.svc.cluster.local + rank : 7 (local_rank: 7) + exitcode : 1 (pid: 10356) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-05-09_23:23:10 + host : t-20260510071136-9xm8l-worker-0.t-20260510071136-9xm8l-worker.mlplatform-customtask.svc.cluster.local + rank : 5 (local_rank: 5) + exitcode : 1 (pid: 10354) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[rank2]:[W509 23:29:26.279125256 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank4]:[W509 23:29:26.547950053 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank1]:[W509 23:29:26.549756211 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank5]:[W509 23:29:26.553080349 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank7]:[W509 23:29:26.563156887 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank6]:[W509 23:29:26.564624231 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank3]:[W509 23:29:26.565890812 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank2]:[W509 23:39:26.326974132 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:43122, remote=[localhost]:29672) returned 0, likely a timeout +[rank2]:[W509 23:39:26.327072920 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:43122, remote=[localhost]:29672) timed out after 600000ms +[rank2]: Traceback (most recent call last): +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank2]: main() +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank2]: dist.barrier() +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank2]: return func(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank2]: work = group.barrier(opts=opts) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: torch.distributed.DistBackendError: [2] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank2]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank2]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f731e814818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank2]: frame #1: + 0x10b6c9e (0x7f736e733c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f7372dcfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f7372dd0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f7372d7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f7372d7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f7372d7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f731f54717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f731f5532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #9: + 0xcd3cb3 (0x7f731f558cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f731f5594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f731f56913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #12: + 0x56e465f (0x7f7372d6165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #13: + 0x56f3cde (0x7f7372d70cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #14: + 0x4dbe9d0 (0x7f737243b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #15: + 0x57029b2 (0x7f7372d7f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #16: + 0x570376d (0x7f7372d8076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #17: + 0xc6875f (0x7f737af8975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank2]: frame #18: + 0x37811d (0x7f737a69911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank2]: frame #19: /usr/bin/python() [0x58208f] +[rank2]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank2]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank2]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank2]: frame #23: /usr/bin/python() [0x608b42] +[rank2]: frame #24: /usr/bin/python() [0x6b4e93] +[rank2]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank2]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank2]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank2]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank2]: frame #29: + 0x2a1ca (0x7f7386fa81ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank2]: frame #30: __libc_start_main + 0x8b (0x7f7386fa828b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank2]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank2]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank5]:[W509 23:39:26.625033665 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:43184, remote=[localhost]:29672) returned 0, likely a timeout +[rank5]:[W509 23:39:26.625109123 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:43184, remote=[localhost]:29672) timed out after 600000ms +[rank1]:[W509 23:39:26.627365261 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:43164, remote=[localhost]:29672) returned 0, likely a timeout +[rank1]:[W509 23:39:26.627430597 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:43164, remote=[localhost]:29672) timed out after 600000ms +[rank5]: Traceback (most recent call last): +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank5]: main() +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank5]: dist.barrier() +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank5]: return func(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank5]: work = group.barrier(opts=opts) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: torch.distributed.DistBackendError: [5] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank5]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank5]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f2bc312f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank5]: frame #1: + 0x10b6c9e (0x7f2c1304ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f2c176eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f2c176ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f2c1768e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f2c1768e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f2c1768e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f2bc3e6217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f2bc3e6e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #9: + 0xcd3cb3 (0x7f2bc3e73cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f2bc3e744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f2bc3e8413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #12: + 0x56e465f (0x7f2c1767c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #13: + 0x56f3cde (0x7f2c1768bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #14: + 0x4dbe9d0 (0x7f2c16d569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #15: + 0x57029b2 (0x7f2c1769a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #16: + 0x570376d (0x7f2c1769b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #17: + 0xc6875f (0x7f2c1f8a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank5]: frame #18: + 0x37811d (0x7f2c1efb411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank5]: frame #19: /usr/bin/python() [0x58208f] +[rank5]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank5]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank5]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank5]: frame #23: /usr/bin/python() [0x608b42] +[rank5]: frame #24: /usr/bin/python() [0x6b4e93] +[rank5]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank5]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank5]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank5]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank5]: frame #29: + 0x2a1ca (0x7f2c2b8551ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank5]: frame #30: __libc_start_main + 0x8b (0x7f2c2b85528b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank5]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank5]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank1]: Traceback (most recent call last): +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank1]: main() +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank1]: dist.barrier() +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank1]: return func(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank1]: work = group.barrier(opts=opts) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: torch.distributed.DistBackendError: [1] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank1]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank1]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7fe346087818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank1]: frame #1: + 0x10b6c9e (0x7fe39604ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7fe39a6eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7fe39a6ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fe39a68e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fe39a68e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fe39a68e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7fe346e6217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7fe346e6e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #9: + 0xcd3cb3 (0x7fe346e73cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7fe346e744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7fe346e8413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #12: + 0x56e465f (0x7fe39a67c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #13: + 0x56f3cde (0x7fe39a68bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #14: + 0x4dbe9d0 (0x7fe399d569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #15: + 0x57029b2 (0x7fe39a69a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #16: + 0x570376d (0x7fe39a69b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #17: + 0xc6875f (0x7fe3a28a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank1]: frame #18: + 0x37811d (0x7fe3a1fb411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank1]: frame #19: /usr/bin/python() [0x58208f] +[rank1]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank1]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank1]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank1]: frame #23: /usr/bin/python() [0x608b42] +[rank1]: frame #24: /usr/bin/python() [0x6b4e93] +[rank1]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank1]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank1]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank1]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank1]: frame #29: + 0x2a1ca (0x7fe3ae8181ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank1]: frame #30: __libc_start_main + 0x8b (0x7fe3ae81828b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank1]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank1]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank4]:[W509 23:39:26.652030954 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:43196, remote=[localhost]:29672) returned 0, likely a timeout +[rank4]:[W509 23:39:26.652095716 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:43196, remote=[localhost]:29672) timed out after 600000ms +W0509 23:39:26.610000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10349 closing signal SIGTERM +W0509 23:39:26.611000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10350 closing signal SIGTERM +W0509 23:39:26.611000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10352 closing signal SIGTERM +W0509 23:39:26.611000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10353 closing signal SIGTERM +W0509 23:39:26.612000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10354 closing signal SIGTERM +W0509 23:39:26.612000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10355 closing signal SIGTERM +W0509 23:39:26.612000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10356 closing signal SIGTERM +E0509 23:39:26.988000 10282 torch/distributed/elastic/multiprocessing/api.py:870] failed (exitcode: 1) local_rank: 2 (pid: 10351) of binary: /usr/bin/python +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +train.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-05-09_23:39:26 + host : t-20260510072327-8pfrg-worker-0.t-20260510072327-8pfrg-worker.mlplatform-customtask.svc.cluster.local + rank : 2 (local_rank: 2) + exitcode : 1 (pid: 10351) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[rank5]:[W509 23:41:14.519033442 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank2]:[W509 23:41:14.548421488 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank4]:[W509 23:41:14.577756706 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank1]:[W509 23:41:14.580495059 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank3]:[W509 23:41:14.589546292 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank6]:[W509 23:41:14.602435943 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank7]:[W509 23:41:14.617784622 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank5]:[W509 23:51:14.545085137 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:50164, remote=[localhost]:29672) returned 0, likely a timeout +[rank5]:[W509 23:51:14.545166172 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:50164, remote=[localhost]:29672) timed out after 600000ms +[rank5]: Traceback (most recent call last): +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank5]: main() +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank5]: dist.barrier() +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank5]: return func(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank5]: work = group.barrier(opts=opts) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: torch.distributed.DistBackendError: [5] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank5]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank5]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f62d816c818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank5]: frame #1: + 0x10b6c9e (0x7f6328133c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f632c7cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f632c7d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f632c77392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f632c77392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f632c77392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f62d8f4717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f62d8f532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #9: + 0xcd3cb3 (0x7f62d8f58cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f62d8f594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f62d8f6913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #12: + 0x56e465f (0x7f632c76165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #13: + 0x56f3cde (0x7f632c770cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #14: + 0x4dbe9d0 (0x7f632be3b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #15: + 0x57029b2 (0x7f632c77f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #16: + 0x570376d (0x7f632c78076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #17: + 0xc6875f (0x7f633498975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank5]: frame #18: + 0x37811d (0x7f633409911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank5]: frame #19: /usr/bin/python() [0x58208f] +[rank5]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank5]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank5]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank5]: frame #23: /usr/bin/python() [0x608b42] +[rank5]: frame #24: /usr/bin/python() [0x6b4e93] +[rank5]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank5]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank5]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank5]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank5]: frame #29: + 0x2a1ca (0x7f63409801ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank5]: frame #30: __libc_start_main + 0x8b (0x7f634098028b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank5]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank5]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank2]:[W509 23:51:14.575952304 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:50174, remote=[localhost]:29672) returned 0, likely a timeout +[rank2]:[W509 23:51:14.576033597 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:50174, remote=[localhost]:29672) timed out after 600000ms +[rank2]: Traceback (most recent call last): +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank2]: main() +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank2]: dist.barrier() +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank2]: return func(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank2]: work = group.barrier(opts=opts) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: torch.distributed.DistBackendError: [2] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank2]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank2]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7fbcf2414818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank2]: frame #1: + 0x10b6c9e (0x7fbd42333c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7fbd469cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7fbd469d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fbd4697392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fbd4697392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fbd4697392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7fbcf314717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7fbcf31532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #9: + 0xcd3cb3 (0x7fbcf3158cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7fbcf31594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7fbcf316913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #12: + 0x56e465f (0x7fbd4696165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #13: + 0x56f3cde (0x7fbd46970cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #14: + 0x4dbe9d0 (0x7fbd4603b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #15: + 0x57029b2 (0x7fbd4697f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #16: + 0x570376d (0x7fbd4698076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #17: + 0xc6875f (0x7fbd4eb8975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank2]: frame #18: + 0x37811d (0x7fbd4e29911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank2]: frame #19: /usr/bin/python() [0x58208f] +[rank2]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank2]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank2]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank2]: frame #23: /usr/bin/python() [0x608b42] +[rank2]: frame #24: /usr/bin/python() [0x6b4e93] +[rank2]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank2]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank2]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank2]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank2]: frame #29: + 0x2a1ca (0x7fbd5ab8e1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank2]: frame #30: __libc_start_main + 0x8b (0x7fbd5ab8e28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank2]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank2]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank3]:[W509 23:51:14.642482701 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:50166, remote=[localhost]:29672) returned 0, likely a timeout +[rank3]:[W509 23:51:14.642567327 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:50166, remote=[localhost]:29672) timed out after 600000ms +[rank3]: Traceback (most recent call last): +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank3]: main() +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank3]: dist.barrier() +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank3]: return func(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank3]: work = group.barrier(opts=opts) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: torch.distributed.DistBackendError: [3] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank3]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank3]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f352292f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank3]: frame #1: + 0x10b6c9e (0x7f357284ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f3576eeae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f3576eebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f3576e8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f3576e8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f3576e8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f352366217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f352366e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #9: + 0xcd3cb3 (0x7f3523673cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f35236744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f352368413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #12: + 0x56e465f (0x7f3576e7c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #13: + 0x56f3cde (0x7f3576e8bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #14: + 0x4dbe9d0 (0x7f35765569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #15: + 0x57029b2 (0x7f3576e9a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #16: + 0x570376d (0x7f3576e9b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #17: + 0xc6875f (0x7f357f0a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank3]: frame #18: + 0x37811d (0x7f357e7b411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank3]: frame #19: /usr/bin/python() [0x58208f] +[rank3]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank3]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank3]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank3]: frame #23: /usr/bin/python() [0x608b42] +[rank3]: frame #24: /usr/bin/python() [0x6b4e93] +[rank3]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank3]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank3]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank3]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank3]: frame #29: + 0x2a1ca (0x7f358b0861ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank3]: frame #30: __libc_start_main + 0x8b (0x7f358b08628b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank3]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank3]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank7]:[W509 23:51:14.676712670 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:50192, remote=[localhost]:29672) returned 0, likely a timeout +[rank7]:[W509 23:51:14.676761557 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:50192, remote=[localhost]:29672) timed out after 600000ms +[rank4]:[W509 23:51:14.682321517 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:50168, remote=[localhost]:29672) returned 0, likely a timeout +[rank4]:[W509 23:51:14.682383595 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:50168, remote=[localhost]:29672) timed out after 600000ms +[rank1]:[W509 23:51:14.684039476 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:50162, remote=[localhost]:29672) returned 0, likely a timeout +[rank1]:[W509 23:51:14.684099637 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:50162, remote=[localhost]:29672) timed out after 600000ms +[rank7]: Traceback (most recent call last): +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank7]: main() +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank7]: dist.barrier() +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank7]: return func(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank7]: work = group.barrier(opts=opts) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: torch.distributed.DistBackendError: [7] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank7]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank7]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f0baaf6c818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank7]: frame #1: + 0x10b6c9e (0x7f0bfaf33c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f0bff5cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f0bff5d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f0bff57392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f0bff57392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f0bff57392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f0babd4717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f0babd532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #9: + 0xcd3cb3 (0x7f0babd58cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f0babd594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f0babd6913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #12: + 0x56e465f (0x7f0bff56165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #13: + 0x56f3cde (0x7f0bff570cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #14: + 0x4dbe9d0 (0x7f0bfec3b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #15: + 0x57029b2 (0x7f0bff57f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #16: + 0x570376d (0x7f0bff58076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #17: + 0xc6875f (0x7f0c0778975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank7]: frame #18: + 0x37811d (0x7f0c06e9911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank7]: frame #19: /usr/bin/python() [0x58208f] +[rank7]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank7]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank7]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank7]: frame #23: /usr/bin/python() [0x608b42] +[rank7]: frame #24: /usr/bin/python() [0x6b4e93] +[rank7]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank7]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank7]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank7]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank7]: frame #29: + 0x2a1ca (0x7f0c137741ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank7]: frame #30: __libc_start_main + 0x8b (0x7f0c1377428b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank7]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank7]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank6]:[W509 23:51:14.686928165 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:50172, remote=[localhost]:29672) returned 0, likely a timeout +[rank6]:[W509 23:51:14.686988637 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:50172, remote=[localhost]:29672) timed out after 600000ms +[rank4]: Traceback (most recent call last): +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank4]: main() +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank4]: dist.barrier() +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank4]: return func(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank4]: work = group.barrier(opts=opts) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: torch.distributed.DistBackendError: [4] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank4]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank4]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7fb2da814818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank4]: frame #1: + 0x10b6c9e (0x7fb32a733c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7fb32edcfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7fb32edd0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fb32ed7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fb32ed7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fb32ed7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7fb2db54717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7fb2db5532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #9: + 0xcd3cb3 (0x7fb2db558cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7fb2db5594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7fb2db56913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #12: + 0x56e465f (0x7fb32ed6165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #13: + 0x56f3cde (0x7fb32ed70cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #14: + 0x4dbe9d0 (0x7fb32e43b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #15: + 0x57029b2 (0x7fb32ed7f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #16: + 0x570376d (0x7fb32ed8076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #17: + 0xc6875f (0x7fb336f8975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank4]: frame #18: + 0x37811d (0x7fb33669911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank4]: frame #19: /usr/bin/python() [0x58208f] +[rank4]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank4]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank4]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank4]: frame #23: /usr/bin/python() [0x608b42] +[rank4]: frame #24: /usr/bin/python() [0x6b4e93] +[rank4]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank4]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank4]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank4]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank4]: frame #29: + 0x2a1ca (0x7fb342f951ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank4]: frame #30: __libc_start_main + 0x8b (0x7fb342f9528b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank4]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank4]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank1]: Traceback (most recent call last): +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank1]: main() +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank1]: dist.barrier() +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank1]: return func(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank1]: work = group.barrier(opts=opts) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: torch.distributed.DistBackendError: [1] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank1]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank1]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f6cb156c818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank1]: frame #1: + 0x10b6c9e (0x7f6d01533c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f6d05bcfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f6d05bd0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f6d05b7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f6d05b7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f6d05b7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f6cb234717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f6cb23532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #9: + 0xcd3cb3 (0x7f6cb2358cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f6cb23594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f6cb236913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #12: + 0x56e465f (0x7f6d05b6165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #13: + 0x56f3cde (0x7f6d05b70cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #14: + 0x4dbe9d0 (0x7f6d0523b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #15: + 0x57029b2 (0x7f6d05b7f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #16: + 0x570376d (0x7f6d05b8076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #17: + 0xc6875f (0x7f6d0dd8975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank1]: frame #18: + 0x37811d (0x7f6d0d49911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank1]: frame #19: /usr/bin/python() [0x58208f] +[rank1]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank1]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank1]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank1]: frame #23: /usr/bin/python() [0x608b42] +[rank1]: frame #24: /usr/bin/python() [0x6b4e93] +[rank1]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank1]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank1]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank1]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank1]: frame #29: + 0x2a1ca (0x7f6d19d4c1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank1]: frame #30: __libc_start_main + 0x8b (0x7f6d19d4c28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank1]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank1]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank6]: Traceback (most recent call last): +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank6]: main() +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank6]: dist.barrier() +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank6]: return func(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank6]: work = group.barrier(opts=opts) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: torch.distributed.DistBackendError: [6] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank6]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank6]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f57a5b6c818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank6]: frame #1: + 0x10b6c9e (0x7f57f5b33c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f57fa1cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f57fa1d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f57fa17392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f57fa17392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f57fa17392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f57a694717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f57a69532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #9: + 0xcd3cb3 (0x7f57a6958cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f57a69594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f57a696913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #12: + 0x56e465f (0x7f57fa16165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #13: + 0x56f3cde (0x7f57fa170cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #14: + 0x4dbe9d0 (0x7f57f983b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #15: + 0x57029b2 (0x7f57fa17f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #16: + 0x570376d (0x7f57fa18076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #17: + 0xc6875f (0x7f580238975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank6]: frame #18: + 0x37811d (0x7f5801a9911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank6]: frame #19: /usr/bin/python() [0x58208f] +[rank6]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank6]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank6]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank6]: frame #23: /usr/bin/python() [0x608b42] +[rank6]: frame #24: /usr/bin/python() [0x6b4e93] +[rank6]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank6]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank6]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank6]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank6]: frame #29: + 0x2a1ca (0x7f580e3041ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank6]: frame #30: __libc_start_main + 0x8b (0x7f580e30428b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank6]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank6]: . This may indicate a possible application crash on rank 0 or a network set up issue. +W0509 23:51:14.723000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10349 closing signal SIGTERM +W0509 23:51:14.724000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10350 closing signal SIGTERM +W0509 23:51:14.724000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10352 closing signal SIGTERM +W0509 23:51:14.725000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10353 closing signal SIGTERM +W0509 23:51:14.725000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10355 closing signal SIGTERM +W0509 23:51:14.725000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10356 closing signal SIGTERM +E0509 23:51:15.016000 10282 torch/distributed/elastic/multiprocessing/api.py:870] failed (exitcode: 1) local_rank: 2 (pid: 10351) of binary: /usr/bin/python +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +train.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2026-05-09_23:51:14 + host : t-20260510073943-nmsst-worker-0.t-20260510073943-nmsst-worker.mlplatform-customtask.svc.cluster.local + rank : 5 (local_rank: 5) + exitcode : 1 (pid: 10354) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-05-09_23:51:14 + host : t-20260510073943-nmsst-worker-0.t-20260510073943-nmsst-worker.mlplatform-customtask.svc.cluster.local + rank : 2 (local_rank: 2) + exitcode : 1 (pid: 10351) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[rank1]:[W509 23:53:02.998913492 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank4]:[W509 23:53:02.104153470 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank2]:[W509 23:53:02.196232154 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank7]:[W509 23:53:02.197080961 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank6]:[W509 23:53:02.215305874 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank5]:[W509 23:53:03.219501313 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank3]:[W509 23:53:03.232972467 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank1]:[W510 00:03:02.062031797 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:44548, remote=[localhost]:29672) returned 0, likely a timeout +[rank1]:[W510 00:03:02.062133827 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:44548, remote=[localhost]:29672) timed out after 600000ms +[rank1]: Traceback (most recent call last): +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank1]: main() +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank1]: dist.barrier() +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank1]: return func(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank1]: work = group.barrier(opts=opts) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: torch.distributed.DistBackendError: [1] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank1]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank1]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7fb97916c818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank1]: frame #1: + 0x10b6c9e (0x7fb9c9133c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7fb9cd7cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7fb9cd7d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fb9cd77392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fb9cd77392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fb9cd77392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7fb979f4717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7fb979f532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #9: + 0xcd3cb3 (0x7fb979f58cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7fb979f594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7fb979f6913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #12: + 0x56e465f (0x7fb9cd76165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #13: + 0x56f3cde (0x7fb9cd770cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #14: + 0x4dbe9d0 (0x7fb9cce3b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #15: + 0x57029b2 (0x7fb9cd77f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #16: + 0x570376d (0x7fb9cd78076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #17: + 0xc6875f (0x7fb9d598975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank1]: frame #18: + 0x37811d (0x7fb9d509911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank1]: frame #19: /usr/bin/python() [0x58208f] +[rank1]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank1]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank1]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank1]: frame #23: /usr/bin/python() [0x608b42] +[rank1]: frame #24: /usr/bin/python() [0x6b4e93] +[rank1]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank1]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank1]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank1]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank1]: frame #29: + 0x2a1ca (0x7fb9e194a1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank1]: frame #30: __libc_start_main + 0x8b (0x7fb9e194a28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank1]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank1]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank4]:[W510 00:03:02.136037266 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:44606, remote=[localhost]:29672) returned 0, likely a timeout +[rank4]:[W510 00:03:02.136110212 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:44606, remote=[localhost]:29672) timed out after 600000ms +[rank4]: Traceback (most recent call last): +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank4]: main() +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank4]: dist.barrier() +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank4]: return func(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank4]: work = group.barrier(opts=opts) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: torch.distributed.DistBackendError: [4] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank4]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank4]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f872e72f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank4]: frame #1: + 0x10b6c9e (0x7f877e64ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f8782ceae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f8782cebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f8782c8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f8782c8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f8782c8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f872f46217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f872f46e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #9: + 0xcd3cb3 (0x7f872f473cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f872f4744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f872f48413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #12: + 0x56e465f (0x7f8782c7c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #13: + 0x56f3cde (0x7f8782c8bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #14: + 0x4dbe9d0 (0x7f87823569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #15: + 0x57029b2 (0x7f8782c9a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #16: + 0x570376d (0x7f8782c9b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #17: + 0xc6875f (0x7f878aea475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank4]: frame #18: + 0x37811d (0x7f878a5b411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank4]: frame #19: /usr/bin/python() [0x58208f] +[rank4]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank4]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank4]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank4]: frame #23: /usr/bin/python() [0x608b42] +[rank4]: frame #24: /usr/bin/python() [0x6b4e93] +[rank4]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank4]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank4]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank4]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank4]: frame #29: + 0x2a1ca (0x7f8796e621ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank4]: frame #30: __libc_start_main + 0x8b (0x7f8796e6228b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank4]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank4]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank6]:[W510 00:03:03.253263674 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:44608, remote=[localhost]:29672) returned 0, likely a timeout +[rank6]:[W510 00:03:03.253338771 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:44608, remote=[localhost]:29672) timed out after 600000ms +[rank2]:[W510 00:03:03.262282342 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:44564, remote=[localhost]:29672) returned 0, likely a timeout +[rank2]:[W510 00:03:03.262344860 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:44564, remote=[localhost]:29672) timed out after 600000ms +[rank6]: Traceback (most recent call last): +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank6]: main() +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank6]: dist.barrier() +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank6]: return func(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank6]: work = group.barrier(opts=opts) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: torch.distributed.DistBackendError: [6] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank6]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank6]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f25bd72f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank6]: frame #1: + 0x10b6c9e (0x7f260d64ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f2611ceae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f2611cebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f2611c8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f2611c8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f2611c8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f25be46217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f25be46e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #9: + 0xcd3cb3 (0x7f25be473cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f25be4744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f25be48413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #12: + 0x56e465f (0x7f2611c7c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #13: + 0x56f3cde (0x7f2611c8bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #14: + 0x4dbe9d0 (0x7f26113569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #15: + 0x57029b2 (0x7f2611c9a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #16: + 0x570376d (0x7f2611c9b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #17: + 0xc6875f (0x7f2619ea475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank6]: frame #18: + 0x37811d (0x7f26195b411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank6]: frame #19: /usr/bin/python() [0x58208f] +[rank6]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank6]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank6]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank6]: frame #23: /usr/bin/python() [0x608b42] +[rank6]: frame #24: /usr/bin/python() [0x6b4e93] +[rank6]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank6]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank6]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank6]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank6]: frame #29: + 0x2a1ca (0x7f2625ef21ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank6]: frame #30: __libc_start_main + 0x8b (0x7f2625ef228b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank6]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank6]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank2]: Traceback (most recent call last): +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank2]: main() +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank2]: dist.barrier() +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank2]: return func(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank2]: work = group.barrier(opts=opts) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: torch.distributed.DistBackendError: [2] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank2]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank2]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f8b9192f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank2]: frame #1: + 0x10b6c9e (0x7f8be184ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f8be5eeae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f8be5eebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f8be5e8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f8be5e8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f8be5e8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f8b9266217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f8b9266e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #9: + 0xcd3cb3 (0x7f8b92673cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f8b926744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f8b9268413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #12: + 0x56e465f (0x7f8be5e7c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #13: + 0x56f3cde (0x7f8be5e8bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #14: + 0x4dbe9d0 (0x7f8be55569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #15: + 0x57029b2 (0x7f8be5e9a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #16: + 0x570376d (0x7f8be5e9b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #17: + 0xc6875f (0x7f8bee0a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank2]: frame #18: + 0x37811d (0x7f8bed7b411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank2]: frame #19: /usr/bin/python() [0x58208f] +[rank2]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank2]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank2]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank2]: frame #23: /usr/bin/python() [0x608b42] +[rank2]: frame #24: /usr/bin/python() [0x6b4e93] +[rank2]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank2]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank2]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank2]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank2]: frame #29: + 0x2a1ca (0x7f8bfa0801ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank2]: frame #30: __libc_start_main + 0x8b (0x7f8bfa08028b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank2]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank2]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank7]:[W510 00:03:03.303044044 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:44574, remote=[localhost]:29672) returned 0, likely a timeout +[rank7]:[W510 00:03:03.303109197 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:44574, remote=[localhost]:29672) timed out after 600000ms +[rank3]:[W510 00:03:03.305720433 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:44602, remote=[localhost]:29672) returned 0, likely a timeout +[rank3]:[W510 00:03:03.305784110 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:44602, remote=[localhost]:29672) timed out after 600000ms +[rank7]: Traceback (most recent call last): +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank7]: main() +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank7]: dist.barrier() +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank7]: return func(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank7]: work = group.barrier(opts=opts) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: torch.distributed.DistBackendError: [7] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank7]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank7]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f4c1192f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank7]: frame #1: + 0x10b6c9e (0x7f4c6184ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f4c65eeae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f4c65eebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f4c65e8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f4c65e8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f4c65e8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f4c1266217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f4c1266e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #9: + 0xcd3cb3 (0x7f4c12673cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f4c126744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f4c1268413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #12: + 0x56e465f (0x7f4c65e7c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #13: + 0x56f3cde (0x7f4c65e8bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #14: + 0x4dbe9d0 (0x7f4c655569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #15: + 0x57029b2 (0x7f4c65e9a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #16: + 0x570376d (0x7f4c65e9b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #17: + 0xc6875f (0x7f4c6e0a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank7]: frame #18: + 0x37811d (0x7f4c6d7b411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank7]: frame #19: /usr/bin/python() [0x58208f] +[rank7]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank7]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank7]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank7]: frame #23: /usr/bin/python() [0x608b42] +[rank7]: frame #24: /usr/bin/python() [0x6b4e93] +[rank7]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank7]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank7]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank7]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank7]: frame #29: + 0x2a1ca (0x7f4c7a0a21ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank7]: frame #30: __libc_start_main + 0x8b (0x7f4c7a0a228b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank7]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank7]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank3]: Traceback (most recent call last): +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank3]: main() +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank3]: dist.barrier() +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank3]: return func(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank3]: work = group.barrier(opts=opts) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: torch.distributed.DistBackendError: [3] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank3]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank3]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f2a21d2f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank3]: frame #1: + 0x10b6c9e (0x7f2a71c4ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f2a762eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f2a762ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f2a7628e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f2a7628e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f2a7628e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f2a22a6217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f2a22a6e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #9: + 0xcd3cb3 (0x7f2a22a73cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f2a22a744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f2a22a8413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #12: + 0x56e465f (0x7f2a7627c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #13: + 0x56f3cde (0x7f2a7628bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #14: + 0x4dbe9d0 (0x7f2a759569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #15: + 0x57029b2 (0x7f2a7629a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #16: + 0x570376d (0x7f2a7629b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #17: + 0xc6875f (0x7f2a7e4a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank3]: frame #18: + 0x37811d (0x7f2a7dbb411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank3]: frame #19: /usr/bin/python() [0x58208f] +[rank3]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank3]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank3]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank3]: frame #23: /usr/bin/python() [0x608b42] +[rank3]: frame #24: /usr/bin/python() [0x6b4e93] +[rank3]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank3]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank3]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank3]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank3]: frame #29: + 0x2a1ca (0x7f2a8a4dd1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank3]: frame #30: __libc_start_main + 0x8b (0x7f2a8a4dd28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank3]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank3]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank5]:[W510 00:03:03.323975059 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:44590, remote=[localhost]:29672) returned 0, likely a timeout +[rank5]:[W510 00:03:03.324082445 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:44590, remote=[localhost]:29672) timed out after 600000ms +[rank5]: Traceback (most recent call last): +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank5]: main() +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank5]: dist.barrier() +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank5]: return func(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank5]: work = group.barrier(opts=opts) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: torch.distributed.DistBackendError: [5] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank5]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank5]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7fb7d696c818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank5]: frame #1: + 0x10b6c9e (0x7fb826933c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7fb82afcfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7fb82afd0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fb82af7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fb82af7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fb82af7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7fb7d774717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7fb7d77532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #9: + 0xcd3cb3 (0x7fb7d7758cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7fb7d77594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7fb7d776913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #12: + 0x56e465f (0x7fb82af6165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #13: + 0x56f3cde (0x7fb82af70cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #14: + 0x4dbe9d0 (0x7fb82a63b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #15: + 0x57029b2 (0x7fb82af7f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #16: + 0x570376d (0x7fb82af8076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #17: + 0xc6875f (0x7fb83318975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank5]: frame #18: + 0x37811d (0x7fb83289911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank5]: frame #19: /usr/bin/python() [0x58208f] +[rank5]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank5]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank5]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank5]: frame #23: /usr/bin/python() [0x608b42] +[rank5]: frame #24: /usr/bin/python() [0x6b4e93] +[rank5]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank5]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank5]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank5]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank5]: frame #29: + 0x2a1ca (0x7fb83f14d1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank5]: frame #30: __libc_start_main + 0x8b (0x7fb83f14d28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank5]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank5]: . This may indicate a possible application crash on rank 0 or a network set up issue. +W0510 00:03:03.242000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10349 closing signal SIGTERM +W0510 00:03:03.242000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10351 closing signal SIGTERM +W0510 00:03:03.243000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10352 closing signal SIGTERM +W0510 00:03:03.243000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10354 closing signal SIGTERM +W0510 00:03:03.244000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10355 closing signal SIGTERM +W0510 00:03:03.244000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10356 closing signal SIGTERM +E0510 00:03:03.522000 10282 torch/distributed/elastic/multiprocessing/api.py:870] failed (exitcode: 1) local_rank: 1 (pid: 10350) of binary: /usr/bin/python +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +train.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2026-05-10_00:03:03 + host : t-20260510075131-bwnv8-worker-0.t-20260510075131-bwnv8-worker.mlplatform-customtask.svc.cluster.local + rank : 4 (local_rank: 4) + exitcode : 1 (pid: 10353) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-05-10_00:03:03 + host : t-20260510075131-bwnv8-worker-0.t-20260510075131-bwnv8-worker.mlplatform-customtask.svc.cluster.local + rank : 1 (local_rank: 1) + exitcode : 1 (pid: 10350) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[rank1]:[W510 00:10:58.789659999 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank4]:[W510 00:10:58.799531720 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank3]:[W510 00:10:58.799891718 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank7]:[W510 00:10:58.810311209 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank2]:[W510 00:10:58.813732419 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank5]:[W510 00:10:58.817222521 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank6]:[W510 00:10:58.844943621 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id. +[rank1]:[W510 00:20:58.837371467 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:34088, remote=[localhost]:29672) returned 0, likely a timeout +[rank1]:[W510 00:20:58.837462700 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:34088, remote=[localhost]:29672) timed out after 600000ms +[rank1]: Traceback (most recent call last): +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank1]: main() +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank1]: dist.barrier() +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank1]: return func(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank1]: work = group.barrier(opts=opts) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: torch.distributed.DistBackendError: [1] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank1]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank1]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7ff866d2f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank1]: frame #1: + 0x10b6c9e (0x7ff8b6c4ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7ff8bb2eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7ff8bb2ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7ff8bb28e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7ff8bb28e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7ff8bb28e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7ff867a6217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7ff867a6e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #9: + 0xcd3cb3 (0x7ff867a73cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7ff867a744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7ff867a8413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank1]: frame #12: + 0x56e465f (0x7ff8bb27c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #13: + 0x56f3cde (0x7ff8bb28bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #14: + 0x4dbe9d0 (0x7ff8ba9569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #15: + 0x57029b2 (0x7ff8bb29a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #16: + 0x570376d (0x7ff8bb29b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank1]: frame #17: + 0xc6875f (0x7ff8c34a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank1]: frame #18: + 0x37811d (0x7ff8c2bb411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank1]: frame #19: /usr/bin/python() [0x58208f] +[rank1]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank1]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank1]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank1]: frame #23: /usr/bin/python() [0x608b42] +[rank1]: frame #24: /usr/bin/python() [0x6b4e93] +[rank1]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank1]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank1]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank1]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank1]: frame #29: + 0x2a1ca (0x7ff8cf4631ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank1]: frame #30: __libc_start_main + 0x8b (0x7ff8cf46328b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank1]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank1]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank5]:[W510 00:20:58.850233299 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:34068, remote=[localhost]:29672) returned 0, likely a timeout +[rank5]:[W510 00:20:58.850297421 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:34068, remote=[localhost]:29672) timed out after 600000ms +[rank5]: Traceback (most recent call last): +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank5]: main() +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank5]: dist.barrier() +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank5]: return func(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank5]: work = group.barrier(opts=opts) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: torch.distributed.DistBackendError: [5] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank5]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank5]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f3b7092f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank5]: frame #1: + 0x10b6c9e (0x7f3bc084ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f3bc4eeae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f3bc4eebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f3bc4e8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f3bc4e8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f3bc4e8e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f3b7166217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f3b7166e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #9: + 0xcd3cb3 (0x7f3b71673cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f3b716744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f3b7168413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank5]: frame #12: + 0x56e465f (0x7f3bc4e7c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #13: + 0x56f3cde (0x7f3bc4e8bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #14: + 0x4dbe9d0 (0x7f3bc45569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #15: + 0x57029b2 (0x7f3bc4e9a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #16: + 0x570376d (0x7f3bc4e9b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank5]: frame #17: + 0xc6875f (0x7f3bcd0a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank5]: frame #18: + 0x37811d (0x7f3bcc7b411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank5]: frame #19: /usr/bin/python() [0x58208f] +[rank5]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank5]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank5]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank5]: frame #23: /usr/bin/python() [0x608b42] +[rank5]: frame #24: /usr/bin/python() [0x6b4e93] +[rank5]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank5]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank5]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank5]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank5]: frame #29: + 0x2a1ca (0x7f3bd90871ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank5]: frame #30: __libc_start_main + 0x8b (0x7f3bd908728b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank5]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank5]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank4]:[W510 00:20:58.880527006 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:34074, remote=[localhost]:29672) returned 0, likely a timeout +[rank4]:[W510 00:20:58.880587860 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:34074, remote=[localhost]:29672) timed out after 600000ms +[rank4]: Traceback (most recent call last): +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank4]: main() +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank4]: dist.barrier() +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank4]: return func(*args, **kwargs) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank4]: work = group.barrier(opts=opts) +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: torch.distributed.DistBackendError: [4] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank4]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank4]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f7e89e14818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank4]: frame #1: + 0x10b6c9e (0x7f7ed9d33c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f7ede3cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f7ede3d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f7ede37392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f7ede37392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f7ede37392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f7e8ab4717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f7e8ab532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #9: + 0xcd3cb3 (0x7f7e8ab58cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f7e8ab594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f7e8ab6913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank4]: frame #12: + 0x56e465f (0x7f7ede36165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #13: + 0x56f3cde (0x7f7ede370cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #14: + 0x4dbe9d0 (0x7f7edda3b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #15: + 0x57029b2 (0x7f7ede37f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #16: + 0x570376d (0x7f7ede38076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank4]: frame #17: + 0xc6875f (0x7f7ee658975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank4]: frame #18: + 0x37811d (0x7f7ee5c9911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank4]: frame #19: /usr/bin/python() [0x58208f] +[rank4]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank4]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank4]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank4]: frame #23: /usr/bin/python() [0x608b42] +[rank4]: frame #24: /usr/bin/python() [0x6b4e93] +[rank4]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank4]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank4]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank4]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank4]: frame #29: + 0x2a1ca (0x7f7ef25bc1ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank4]: frame #30: __libc_start_main + 0x8b (0x7f7ef25bc28b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank4]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank4]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank2]:[W510 00:20:58.890595503 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:34056, remote=[localhost]:29672) returned 0, likely a timeout +[rank2]:[W510 00:20:58.890648173 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:34056, remote=[localhost]:29672) timed out after 600000ms +[rank3]:[W510 00:20:58.900035663 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:34040, remote=[localhost]:29672) returned 0, likely a timeout +[rank3]:[W510 00:20:58.900095932 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:34040, remote=[localhost]:29672) timed out after 600000ms +[rank2]: Traceback (most recent call last): +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank2]: main() +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank2]: dist.barrier() +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank2]: return func(*args, **kwargs) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank2]: work = group.barrier(opts=opts) +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: torch.distributed.DistBackendError: [2] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank2]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank2]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7fb2ca814818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank2]: frame #1: + 0x10b6c9e (0x7fb31a733c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7fb31edcfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7fb31edd0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fb31ed7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fb31ed7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fb31ed7392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7fb2cb54717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7fb2cb5532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #9: + 0xcd3cb3 (0x7fb2cb558cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7fb2cb5594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7fb2cb56913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank2]: frame #12: + 0x56e465f (0x7fb31ed6165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #13: + 0x56f3cde (0x7fb31ed70cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #14: + 0x4dbe9d0 (0x7fb31e43b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #15: + 0x57029b2 (0x7fb31ed7f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #16: + 0x570376d (0x7fb31ed8076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank2]: frame #17: + 0xc6875f (0x7fb326f8975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank2]: frame #18: + 0x37811d (0x7fb32669911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank2]: frame #19: /usr/bin/python() [0x58208f] +[rank2]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank2]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank2]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank2]: frame #23: /usr/bin/python() [0x608b42] +[rank2]: frame #24: /usr/bin/python() [0x6b4e93] +[rank2]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank2]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank2]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank2]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank2]: frame #29: + 0x2a1ca (0x7fb332fc01ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank2]: frame #30: __libc_start_main + 0x8b (0x7fb332fc028b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank2]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank2]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank3]: Traceback (most recent call last): +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank3]: main() +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank3]: dist.barrier() +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank3]: return func(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank3]: work = group.barrier(opts=opts) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: torch.distributed.DistBackendError: [3] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank3]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank3]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f71a0214818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank3]: frame #1: + 0x10b6c9e (0x7f71f0133c9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f71f47cfe83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f71f47d0ffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f71f477392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f71f477392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f71f477392f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f71a0f4717d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f71a0f532bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #9: + 0xcd3cb3 (0x7f71a0f58cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f71a0f594be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f71a0f6913b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank3]: frame #12: + 0x56e465f (0x7f71f476165f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #13: + 0x56f3cde (0x7f71f4770cde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #14: + 0x4dbe9d0 (0x7f71f3e3b9d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #15: + 0x57029b2 (0x7f71f477f9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #16: + 0x570376d (0x7f71f478076d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank3]: frame #17: + 0xc6875f (0x7f71fc98975f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank3]: frame #18: + 0x37811d (0x7f71fc09911d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank3]: frame #19: /usr/bin/python() [0x58208f] +[rank3]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank3]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank3]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank3]: frame #23: /usr/bin/python() [0x608b42] +[rank3]: frame #24: /usr/bin/python() [0x6b4e93] +[rank3]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank3]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank3]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank3]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank3]: frame #29: + 0x2a1ca (0x7f72089c41ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank3]: frame #30: __libc_start_main + 0x8b (0x7f72089c428b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank3]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank3]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank7]:[W510 00:20:58.914599203 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:34018, remote=[localhost]:29672) returned 0, likely a timeout +[rank7]:[W510 00:20:58.914660756 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:34018, remote=[localhost]:29672) timed out after 600000ms +[rank7]: Traceback (most recent call last): +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank7]: main() +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank7]: dist.barrier() +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank7]: return func(*args, **kwargs) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank7]: work = group.barrier(opts=opts) +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: torch.distributed.DistBackendError: [7] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank7]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank7]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7f4d40b2f818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank7]: frame #1: + 0x10b6c9e (0x7f4d90a4ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7f4d950eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7f4d950ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f4d9508e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f4d9508e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7f4d9508e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7f4d4186217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7f4d4186e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #9: + 0xcd3cb3 (0x7f4d41873cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7f4d418744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7f4d4188413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank7]: frame #12: + 0x56e465f (0x7f4d9507c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #13: + 0x56f3cde (0x7f4d9508bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #14: + 0x4dbe9d0 (0x7f4d947569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #15: + 0x57029b2 (0x7f4d9509a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #16: + 0x570376d (0x7f4d9509b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank7]: frame #17: + 0xc6875f (0x7f4d9d2a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank7]: frame #18: + 0x37811d (0x7f4d9c9b411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank7]: frame #19: /usr/bin/python() [0x58208f] +[rank7]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank7]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank7]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank7]: frame #23: /usr/bin/python() [0x608b42] +[rank7]: frame #24: /usr/bin/python() [0x6b4e93] +[rank7]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank7]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank7]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank7]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank7]: frame #29: + 0x2a1ca (0x7f4da92c81ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank7]: frame #30: __libc_start_main + 0x8b (0x7f4da92c828b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank7]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank7]: . This may indicate a possible application crash on rank 0 or a network set up issue. +[rank6]:[W510 00:20:58.949268561 socket.cpp:462] [c10d] waitForInput: poll for socket SocketImpl(fd=5, addr=[localhost]:34032, remote=[localhost]:29672) returned 0, likely a timeout +[rank6]:[W510 00:20:58.949346060 socket.cpp:487] [c10d] waitForInput: socket SocketImpl(fd=5, addr=[localhost]:34032, remote=[localhost]:29672) timed out after 600000ms +[rank6]: Traceback (most recent call last): +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 661, in +[rank6]: main() +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 325, in main +[rank6]: dist.barrier() +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py", line 81, in wrapper +[rank6]: return func(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/distributed/distributed_c10d.py", line 4553, in barrier +[rank6]: work = group.barrier(opts=opts) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: torch.distributed.DistBackendError: [6] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: wait timeout after 600000ms, keys: /default_pg/0//cuda//0 +[rank6]: Exception raised from doWait at /opt/pytorch/pytorch/torch/csrc/distributed/c10d/TCPStore.cpp:572 (most recent call first): +[rank6]: frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x98 (0x7fe6fbe87818 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +[rank6]: frame #1: + 0x10b6c9e (0x7fe74be4ec9e in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #2: c10d::TCPStore::doGet(std::__cxx11::basic_string, std::allocator > const&) + 0x33 (0x7fe7504eae83 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #3: c10d::TCPStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0xab (0x7fe7504ebffb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #4: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fe75048e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #5: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fe75048e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #6: c10d::PrefixStore::get(std::__cxx11::basic_string, std::allocator > const&) + 0x2f (0x7fe75048e92f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #7: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::__cxx11::basic_string, std::allocator > const&, int) + 0x16d (0x7fe6fcc6217d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #8: c10d::ProcessGroupNCCL::initNCCLComm(std::__cxx11::basic_string, std::allocator > const&, c10::Device&, c10d::OpType, int, bool) + 0x3ac (0x7fe6fcc6e2bc in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #9: + 0xcd3cb3 (0x7fe6fcc73cb3 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #10: c10d::ProcessGroupNCCL::allreduce_impl(at::Tensor&, char const*, c10d::AllreduceOptions const&) + 0xee (0x7fe6fcc744be in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #11: c10d::ProcessGroupNCCL::barrier(c10d::BarrierOptions const&) + 0x69b (0x7fe6fcc8413b in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +[rank6]: frame #12: + 0x56e465f (0x7fe75047c65f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #13: + 0x56f3cde (0x7fe75048bcde in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #14: + 0x4dbe9d0 (0x7fe74fb569d0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #15: + 0x57029b2 (0x7fe75049a9b2 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #16: + 0x570376d (0x7fe75049b76d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cpu.so) +[rank6]: frame #17: + 0xc6875f (0x7fe7586a475f in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank6]: frame #18: + 0x37811d (0x7fe757db411d in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +[rank6]: frame #19: /usr/bin/python() [0x58208f] +[rank6]: frame #20: _PyObject_MakeTpCall + 0x13e (0x54924e in /usr/bin/python) +[rank6]: frame #21: _PyEval_EvalFrameDefault + 0xa89 (0x5d73c9 in /usr/bin/python) +[rank6]: frame #22: PyEval_EvalCode + 0x15b (0x5d58eb in /usr/bin/python) +[rank6]: frame #23: /usr/bin/python() [0x608b42] +[rank6]: frame #24: /usr/bin/python() [0x6b4e93] +[rank6]: frame #25: _PyRun_SimpleFileObject + 0x1aa (0x6b4bfa in /usr/bin/python) +[rank6]: frame #26: _PyRun_AnyFileObject + 0x4f (0x6b4a2f in /usr/bin/python) +[rank6]: frame #27: Py_RunMain + 0x3b5 (0x6bca95 in /usr/bin/python) +[rank6]: frame #28: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +[rank6]: frame #29: + 0x2a1ca (0x7fe7646411ca in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank6]: frame #30: __libc_start_main + 0x8b (0x7fe76464128b in /usr/lib/x86_64-linux-gnu/libc.so.6) +[rank6]: frame #31: _start + 0x25 (0x657ce5 in /usr/bin/python) +[rank6]: . This may indicate a possible application crash on rank 0 or a network set up issue. +W0510 00:20:59.220000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10349 closing signal SIGTERM +W0510 00:20:59.221000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10352 closing signal SIGTERM +W0510 00:20:59.221000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10353 closing signal SIGTERM +W0510 00:20:59.222000 10282 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10355 closing signal SIGTERM +E0510 00:20:59.436000 10282 torch/distributed/elastic/multiprocessing/api.py:870] failed (exitcode: 1) local_rank: 1 (pid: 10350) of binary: /usr/bin/python +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +train.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2026-05-10_00:20:59 + host : t-20260510080320-2j68w-worker-0.t-20260510080320-2j68w-worker.mlplatform-customtask.svc.cluster.local + rank : 2 (local_rank: 2) + exitcode : 1 (pid: 10351) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[2]: + time : 2026-05-10_00:20:59 + host : t-20260510080320-2j68w-worker-0.t-20260510080320-2j68w-worker.mlplatform-customtask.svc.cluster.local + rank : 5 (local_rank: 5) + exitcode : 1 (pid: 10354) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[3]: + time : 2026-05-10_00:20:59 + host : t-20260510080320-2j68w-worker-0.t-20260510080320-2j68w-worker.mlplatform-customtask.svc.cluster.local + rank : 7 (local_rank: 7) + exitcode : 1 (pid: 10356) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-05-10_00:20:59 + host : t-20260510080320-2j68w-worker-0.t-20260510080320-2j68w-worker.mlplatform-customtask.svc.cluster.local + rank : 1 (local_rank: 1) + exitcode : 1 (pid: 10350) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ diff --git a/LTA_openwebtext_dualt/logs/owt_dirichlet_len1024_Cv_to_2v_gumbel_sde_watch/infer_lta_owt_t5elf_dirichlet_len1024_Cv_to_2v_mask1_gbs512_b32_8gpu_20k_save1k_gumbelwatch_20260525_step_0003000.log b/LTA_openwebtext_dualt/logs/owt_dirichlet_len1024_Cv_to_2v_gumbel_sde_watch/infer_lta_owt_t5elf_dirichlet_len1024_Cv_to_2v_mask1_gbs512_b32_8gpu_20k_save1k_gumbelwatch_20260525_step_0003000.log new file mode 100644 index 0000000000000000000000000000000000000000..2a9ea6332b1bba812a3ff11cd2d96d49775210a7 --- /dev/null +++ b/LTA_openwebtext_dualt/logs/owt_dirichlet_len1024_Cv_to_2v_gumbel_sde_watch/infer_lta_owt_t5elf_dirichlet_len1024_Cv_to_2v_mask1_gbs512_b32_8gpu_20k_save1k_gumbelwatch_20260525_step_0003000.log @@ -0,0 +1,136 @@ +[watch-gumbel] 2026-05-25_15:35:51 infer runs/lta_owt_t5elf_dirichlet_len1024_Cv_to_2v_mask1_gbs512_b32_8gpu_20k_save1k_gumbelwatch_20260525/step_0003000.pt -> docs/lta_samples/metrics_20260525/owt_t5elf_dirichlet_len1024_Cv_to_2v_mask1_every1k_sde_gumbel_topp0.95_tau1.0_to_0.2_blend_c32100_64200_n128/lta_owt_t5elf_dirichlet_len1024_Cv_to_2v_mask1_gbs512_b32_8gpu_20k_save1k_gumbelwatch_20260525/step_0003000 +[load] runs/lta_owt_t5elf_dirichlet_len1024_Cv_to_2v_mask1_gbs512_b32_8gpu_20k_save1k_gumbelwatch_20260525/step_0003000.pt +[ckpt] step=3000 +[sde] generated 2/128 +[sde] generated 4/128 +[sde] generated 6/128 +[sde] generated 8/128 +[sde] generated 10/128 +[sde] generated 12/128 +[sde] generated 14/128 +[sde] generated 16/128 +[sde] generated 18/128 +[sde] generated 20/128 +[sde] generated 22/128 +[sde] generated 24/128 +[sde] generated 26/128 +[sde] generated 28/128 +[sde] generated 30/128 +[sde] generated 32/128 +[sde] generated 34/128 +[sde] generated 36/128 +[sde] generated 38/128 +[sde] generated 40/128 +[sde] generated 42/128 +[sde] generated 44/128 +[sde] generated 46/128 +[sde] generated 48/128 +[sde] generated 50/128 +[sde] generated 52/128 +[sde] generated 54/128 +[sde] generated 56/128 +[sde] generated 58/128 +[sde] generated 60/128 +[sde] generated 62/128 +[sde] generated 64/128 +[sde] generated 66/128 +[sde] generated 68/128 +[sde] generated 70/128 +[sde] generated 72/128 +[sde] generated 74/128 +[sde] generated 76/128 +[sde] generated 78/128 +[sde] generated 80/128 +[sde] generated 82/128 +[sde] generated 84/128 +[sde] generated 86/128 +[sde] generated 88/128 +[sde] generated 90/128 +[sde] generated 92/128 +[sde] generated 94/128 +[sde] generated 96/128 +[sde] generated 98/128 +[sde] generated 100/128 +[sde] generated 102/128 +[sde] generated 104/128 +[sde] generated 106/128 +[sde] generated 108/128 +[sde] generated 110/128 +[sde] generated 112/128 +[sde] generated 114/128 +[sde] generated 116/128 +[sde] generated 118/128 +[sde] generated 120/128 +[sde] generated 122/128 +[sde] generated 124/128 +[sde] generated 126/128 +[sde] generated 128/128 +[score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard +[summary] { + "type": "summary", + "checkpoint": "runs/lta_owt_t5elf_dirichlet_len1024_Cv_to_2v_mask1_gbs512_b32_8gpu_20k_save1k_gumbelwatch_20260525/step_0003000.pt", + "step": 3000, + "decode": { + "decode_rule": "dirichlet_resample_sde", + "steps": 128, + "model_t_mode": "support_t", + "mean_mode": "endpoint_only", + "anchor_gamma": 1.0, + "endpoint_floor": 0.0, + "concentration_min": 32100.0, + "concentration_max": 64200.0, + "endpoint_temp": 1.45, + "endpoint_temp_start": null, + "endpoint_temp_end": null, + "endpoint_projection": "gumbel_softmax", + "endpoint_top_k": 0, + "endpoint_top_p": 0.95, + "gumbel_tau_start": 1.0, + "gumbel_tau_end": 0.2, + "gumbel_noise_scale_start": 1.0, + "gumbel_noise_scale_end": 1.0, + "ban_special_tokens": false, + "banned_endpoint_ids": [], + "support_power": 1.0, + "semantic_power": 1.0, + "noise_init": "dirichlet", + "noise_sigma": -1.0, + "noise_dirichlet_concentration": 32100.0, + "sde_resample": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 3.0, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.0, + "final_from": "blend_0.5", + "n_samples": 128, + "seed": 20260524 + }, + "raw_genppl": { + "ppl": 4.952944612195395, + "nll_per_token": 1.599982270865479, + "tokens": 104505, + "kept_samples": 128, + "total_samples": 128, + "empty_rate": 0.0, + "skipped_samples": 0 + }, + "stripped_genppl": { + "ppl": 4.912305141565411, + "nll_per_token": 1.5917433105695673, + "tokens": 103284, + "kept_samples": 128, + "total_samples": 128, + "empty_rate": 0.0, + "skipped_samples": 0 + }, + "diversity": { + "sample_entropy": 1.1192433330446419, + "unique_tokens": 237, + "token_count": 131072, + "distinct_1": 0.00180816650390625, + "distinct_2": 0.005452712609970675, + "top_token_mass": 0.5544662475585938 + } +} +[done] docs/lta_samples/metrics_20260525/owt_t5elf_dirichlet_len1024_Cv_to_2v_mask1_every1k_sde_gumbel_topp0.95_tau1.0_to_0.2_blend_c32100_64200_n128/lta_owt_t5elf_dirichlet_len1024_Cv_to_2v_mask1_gbs512_b32_8gpu_20k_save1k_gumbelwatch_20260525/step_0003000/sde_steps128_samples128_scored.jsonl +[watch-gumbel] 2026-05-25_15:42:20 done step_0003000 diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/__init__.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/data/__init__.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/data/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/data/mt19937-testset-1.csv b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/data/mt19937-testset-1.csv new file mode 100644 index 0000000000000000000000000000000000000000..b97bfa66f72f1be31cdb32317905d805fe90f4d5 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/data/mt19937-testset-1.csv @@ -0,0 +1,1001 @@ +seed, 0xdeadbeaf +0, 0xc816921f +1, 0xb3623c6d +2, 0x5fa391bb +3, 0x40178d9 +4, 0x7dcc9811 +5, 0x548eb8e6 +6, 0x92ba3125 +7, 0x65fde68d +8, 0x2f81ec95 +9, 0xbd94f7a2 +10, 0xdc4d9bcc +11, 0xa672bf13 +12, 0xb41113e +13, 0xec7e0066 +14, 0x50239372 +15, 0xd9d66b1d +16, 0xab72a161 +17, 0xddc2e29f +18, 0x7ea29ab4 +19, 0x80d141ba +20, 0xb1c7edf1 +21, 0x44d29203 +22, 0xe224d98 +23, 0x5b3e9d26 +24, 0x14fd567c +25, 0x27d98c96 +26, 0x838779fc +27, 0x92a138a +28, 0x5d08965b +29, 0x531e0ad6 +30, 0x984ee8f4 +31, 0x1ed78539 +32, 0x32bd6d8d +33, 0xc37c8516 +34, 0x9aef5c6b +35, 0x3aacd139 +36, 0xd96ed154 +37, 0x489cd1ed +38, 0x2cba4b3b +39, 0x76c6ae72 +40, 0x2dae02b9 +41, 0x52ac5fd6 +42, 0xc2b5e265 +43, 0x630e6a28 +44, 0x3f560d5d +45, 0x9315bdf3 +46, 0xf1055aba +47, 0x840e42c6 +48, 0xf2099c6b +49, 0x15ff7696 +50, 0x7948d146 +51, 0x97342961 +52, 0x7a7a21c +53, 0xc66f4fb1 +54, 0x23c4103e +55, 0xd7321f98 +56, 0xeb7efb75 +57, 0xe02490b5 +58, 0x2aa02de +59, 0x8bee0bf7 +60, 0xfc2da059 +61, 0xae835034 +62, 0x678f2075 +63, 0x6d03094b +64, 0x56455e05 +65, 0x18b32373 +66, 0x8ff0356b +67, 0x1fe442fb +68, 0x3f1ab6c3 +69, 0xb6fd21b +70, 0xfc310eb2 +71, 0xb19e9a4d +72, 0x17ddee72 +73, 0xfd534251 +74, 0x9e500564 +75, 0x9013a036 +76, 0xcf08f118 +77, 0x6b6d5969 +78, 0x3ccf1977 +79, 0x7cc11497 +80, 0x651c6ac9 +81, 0x4d6b104b +82, 0x9a28314e +83, 0x14c237be +84, 0x9cfc8d52 +85, 0x2947fad5 +86, 0xd71eff49 +87, 0x5188730e +88, 0x4b894614 +89, 0xf4fa2a34 +90, 0x42f7cc69 +91, 0x4089c9e8 +92, 0xbf0bbfe4 +93, 0x3cea65c +94, 0xc6221207 +95, 0x1bb71a8f +96, 0x54843fe7 +97, 0xbc59de4c +98, 0x79c6ee64 +99, 0x14e57a26 +100, 0x68d88fe +101, 0x2b86ef64 +102, 0x8ffff3c1 +103, 0x5bdd573f +104, 0x85671813 +105, 0xefe32ca2 +106, 0x105ded1e +107, 0x90ca2769 +108, 0xb33963ac +109, 0x363fbbc3 +110, 0x3b3763ae +111, 0x1d50ab88 +112, 0xc9ec01eb +113, 0xc8bbeada +114, 0x5d704692 +115, 0x5fd9e40 +116, 0xe61c125 +117, 0x2fe05792 +118, 0xda8afb72 +119, 0x4cbaa653 +120, 0xdd2243df +121, 0x896fd3f5 +122, 0x5bc23db +123, 0xa1c4e807 +124, 0x57d1a24d +125, 0x66503ddc +126, 0xcf7c0838 +127, 0x19e034fc +128, 0x66807450 +129, 0xfc219b3b +130, 0xe8a843e7 +131, 0x9ce61f08 +132, 0x92b950d6 +133, 0xce955ec4 +134, 0xda0d1f0d +135, 0x960c6250 +136, 0x39552432 +137, 0xde845e84 +138, 0xff3b4b11 +139, 0x5d918e6f +140, 0xbb930df2 +141, 0x7cfb0993 +142, 0x5400e1e9 +143, 0x3bfa0954 +144, 0x7e2605fb +145, 0x11941591 +146, 0x887e6994 +147, 0xdc8bed45 +148, 0x45b3fb50 +149, 0xfbdf8358 +150, 0x41507468 +151, 0x34c87166 +152, 0x17f64d77 +153, 0x3bbaf4f8 +154, 0x4f26f37e +155, 0x4a56ebf2 +156, 0x81100f1 +157, 0x96d94eae +158, 0xca88fda5 +159, 0x2eef3a60 +160, 0x952afbd3 +161, 0x2bec88c7 +162, 0x52335c4b +163, 0x8296db8e +164, 0x4da7d00a +165, 0xc00ac899 +166, 0xadff8c72 +167, 0xbecf26cf +168, 0x8835c83c +169, 0x1d13c804 +170, 0xaa940ddc +171, 0x68222cfe +172, 0x4569c0e1 +173, 0x29077976 +174, 0x32d4a5af +175, 0xd31fcdef +176, 0xdc60682b +177, 0x7c95c368 +178, 0x75a70213 +179, 0x43021751 +180, 0x5e52e0a6 +181, 0xf7e190b5 +182, 0xee3e4bb +183, 0x2fe3b150 +184, 0xcf419c07 +185, 0x478a4570 +186, 0xe5c3ea50 +187, 0x417f30a8 +188, 0xf0cfdaa0 +189, 0xd1f7f738 +190, 0x2c70fc23 +191, 0x54fc89f9 +192, 0x444dcf01 +193, 0xec2a002d +194, 0xef0c3a88 +195, 0xde21be9 +196, 0x88ab3296 +197, 0x3028897c +198, 0x264b200b +199, 0xd8ae0706 +200, 0x9eef901a +201, 0xbd1b96e0 +202, 0xea71366c +203, 0x1465b694 +204, 0x5a794650 +205, 0x83df52d4 +206, 0x8262413d +207, 0x5bc148c0 +208, 0xe0ecd80c +209, 0x40649571 +210, 0xb4d2ee5f +211, 0xedfd7d09 +212, 0xa082e25f +213, 0xc62992d1 +214, 0xbc7e65ee +215, 0x5499cf8a +216, 0xac28f775 +217, 0x649840fb +218, 0xd4c54805 +219, 0x1d166ba6 +220, 0xbeb1171f +221, 0x45b66703 +222, 0x78c03349 +223, 0x38d2a6ff +224, 0x935cae8b +225, 0x1d07dc3f +226, 0x6c1ed365 +227, 0x579fc585 +228, 0x1320c0ec +229, 0x632757eb +230, 0xd265a397 +231, 0x70e9b6c2 +232, 0xc81e322c +233, 0xa27153cf +234, 0x2118ba19 +235, 0x514ec400 +236, 0x2bd0ecd6 +237, 0xc3e7dae3 +238, 0xfa39355e +239, 0x48f23cc1 +240, 0xbcf75948 +241, 0x53ccc70c +242, 0x75346423 +243, 0x951181e0 +244, 0x348e90df +245, 0x14365d7f +246, 0xfbc95d7a +247, 0xdc98a9e6 +248, 0xed202df7 +249, 0xa59ec913 +250, 0x6b6e9ae2 +251, 0x1697f265 +252, 0x15d322d0 +253, 0xa2e7ee0a +254, 0x88860b7e +255, 0x455d8b9d +256, 0x2f5c59cb +257, 0xac49c9f1 +258, 0xa6a6a039 +259, 0xc057f56b +260, 0xf1ff1208 +261, 0x5eb8dc9d +262, 0xe6702509 +263, 0xe238b0ed +264, 0x5ae32e3d +265, 0xa88ebbdf +266, 0xef885ae7 +267, 0xafa6d49b +268, 0xc94499e0 +269, 0x1a196325 +270, 0x88938da3 +271, 0x14f4345 +272, 0xd8e33637 +273, 0xa3551bd5 +274, 0x73fe35c7 +275, 0x9561e94b +276, 0xd673bf68 +277, 0x16134872 +278, 0x68c42f9f +279, 0xdf7574c8 +280, 0x8809bab9 +281, 0x1432cf69 +282, 0xafb66bf1 +283, 0xc184aa7b +284, 0xedbf2007 +285, 0xbd420ce1 +286, 0x761033a0 +287, 0xff7e351f +288, 0xd6c3780e +289, 0x5844416f +290, 0xc6c0ee1c +291, 0xd2e147db +292, 0x92ac601a +293, 0x393e846b +294, 0x18196cca +295, 0x54a22be +296, 0x32bab1c4 +297, 0x60365183 +298, 0x64fa342 +299, 0xca24a493 +300, 0xd8cc8b83 +301, 0x3faf102b +302, 0x6e09bb58 +303, 0x812f0ea +304, 0x592c95d8 +305, 0xe45ea4c5 +306, 0x23aebf83 +307, 0xbd9691d4 +308, 0xf47b4baa +309, 0x4ac7b487 +310, 0xcce18803 +311, 0x3377556e +312, 0x3ff8e6b6 +313, 0x99d22063 +314, 0x23250bec +315, 0x4e1f9861 +316, 0x8554249b +317, 0x8635c2fc +318, 0xe8426e8a +319, 0x966c29d8 +320, 0x270b6082 +321, 0x3180a8a1 +322, 0xe7e1668b +323, 0x7f868dc +324, 0xcf4c17cf +325, 0xe31de4d1 +326, 0xc8c8aff4 +327, 0xae8db704 +328, 0x3c928cc2 +329, 0xe12cd48 +330, 0xb33ecd04 +331, 0xb93d7cbe +332, 0x49c69d6a +333, 0x7d3bce64 +334, 0x86bc219 +335, 0x8408233b +336, 0x44dc7479 +337, 0xdf80d538 +338, 0xf3db02c3 +339, 0xbbbd31d7 +340, 0x121281f +341, 0x7521e9a3 +342, 0x8859675a +343, 0x75aa6502 +344, 0x430ed15b +345, 0xecf0a28d +346, 0x659774fd +347, 0xd58a2311 +348, 0x512389a9 +349, 0xff65e1ff +350, 0xb6ddf222 +351, 0xe3458895 +352, 0x8b13cd6e +353, 0xd4a22870 +354, 0xe604c50c +355, 0x27f54f26 +356, 0x8f7f422f +357, 0x9735b4cf +358, 0x414072b0 +359, 0x76a1c6d5 +360, 0xa2208c06 +361, 0x83cd0f61 +362, 0x6c4f7ead +363, 0x6553cf76 +364, 0xeffcf44 +365, 0x7f434a3f +366, 0x9dc364bd +367, 0x3cdf52ed +368, 0xad597594 +369, 0x9c3e211b +370, 0x6c04a33f +371, 0x885dafa6 +372, 0xbbdaca71 +373, 0x7ae5dd5c +374, 0x37675644 +375, 0x251853c6 +376, 0x130b086b +377, 0x143fa54b +378, 0x54cdc282 +379, 0x9faff5b3 +380, 0x502a5c8b +381, 0xd9524550 +382, 0xae221aa6 +383, 0x55cf759b +384, 0x24782da4 +385, 0xd715d815 +386, 0x250ea09a +387, 0x4e0744ac +388, 0x11e15814 +389, 0xabe5f9df +390, 0xc8146350 +391, 0xfba67d9b +392, 0x2b82e42f +393, 0xd4ea96fc +394, 0x5ffc179e +395, 0x1598bafe +396, 0x7fb6d662 +397, 0x1a12a0db +398, 0x450cee4a +399, 0x85f8e12 +400, 0xce71b594 +401, 0xd4bb1d19 +402, 0x968f379d +403, 0x54cc1d52 +404, 0x467e6066 +405, 0x7da5f9a9 +406, 0x70977034 +407, 0x49e65c4b +408, 0xd08570d1 +409, 0x7acdf60b +410, 0xdffa038b +411, 0x9ce14e4c +412, 0x107cbbf8 +413, 0xdd746ca0 +414, 0xc6370a46 +415, 0xe7f83312 +416, 0x373fa9ce +417, 0xd822a2c6 +418, 0x1d4efea6 +419, 0xc53dcadb +420, 0x9b4e898f +421, 0x71daa6bf +422, 0x7a0bc78b +423, 0xd7b86f50 +424, 0x1b8b3286 +425, 0xcf9425dd +426, 0xd5263220 +427, 0x4ea0b647 +428, 0xc767fe64 +429, 0xcfc5e67 +430, 0xcc6a2942 +431, 0xa51eff00 +432, 0x76092e1b +433, 0xf606e80f +434, 0x824b5e20 +435, 0xebb55e14 +436, 0x783d96a6 +437, 0x10696512 +438, 0x17ee510a +439, 0x3ab70a1f +440, 0xcce6b210 +441, 0x8f72f0fb +442, 0xf0610b41 +443, 0x83d01fb5 +444, 0x6b3de36 +445, 0xe4c2e84f +446, 0x9c43bb15 +447, 0xddf2905 +448, 0x7dd63556 +449, 0x3662ca09 +450, 0xfb81f35b +451, 0xc2c8a72a +452, 0x8e93c37 +453, 0xa93da2d4 +454, 0xa03af8f1 +455, 0x8d75159a +456, 0x15f010b0 +457, 0xa296ab06 +458, 0xe55962ba +459, 0xeae700a9 +460, 0xe388964a +461, 0x917f2bec +462, 0x1c203fea +463, 0x792a01ba +464, 0xa93a80ac +465, 0x9eb8a197 +466, 0x56c0bc73 +467, 0xb8f05799 +468, 0xf429a8c8 +469, 0xb92cee42 +470, 0xf8864ec +471, 0x62f2518a +472, 0x3a7bfa3e +473, 0x12e56e6d +474, 0xd7a18313 +475, 0x41fa3899 +476, 0xa09c4956 +477, 0xebcfd94a +478, 0xc485f90b +479, 0x4391ce40 +480, 0x742a3333 +481, 0xc932f9e5 +482, 0x75c6c263 +483, 0x80937f0 +484, 0xcf21833c +485, 0x16027520 +486, 0xd42e669f +487, 0xb0f01fb7 +488, 0xb35896f1 +489, 0x763737a9 +490, 0x1bb20209 +491, 0x3551f189 +492, 0x56bc2602 +493, 0xb6eacf4 +494, 0x42ec4d11 +495, 0x245cc68 +496, 0xc27ac43b +497, 0x9d903466 +498, 0xce3f0c05 +499, 0xb708c31c +500, 0xc0fd37eb +501, 0x95938b2c +502, 0xf20175a7 +503, 0x4a86ee9b +504, 0xbe039a58 +505, 0xd41cabe7 +506, 0x83bc99ba +507, 0x761d60e1 +508, 0x7737cc2e +509, 0x2b82fc4b +510, 0x375aa401 +511, 0xfe9597a0 +512, 0x5543806a +513, 0x44f31238 +514, 0x7df31538 +515, 0x74cfa770 +516, 0x8755d881 +517, 0x1fde665a +518, 0xda8bf315 +519, 0x973d8e95 +520, 0x72205228 +521, 0x8fe59717 +522, 0x7bb90b34 +523, 0xef6ed945 +524, 0x16fd4a38 +525, 0x5db44de1 +526, 0xf09f93b3 +527, 0xe84824cc +528, 0x945bb50e +529, 0xd0be4aa5 +530, 0x47c277c2 +531, 0xd3800c28 +532, 0xac1c33ec +533, 0xd3dacce +534, 0x811c8387 +535, 0x6761b36 +536, 0x70d3882f +537, 0xd6e62e3a +538, 0xea25daa2 +539, 0xb07f39d1 +540, 0x391d89d7 +541, 0x84b6fb5e +542, 0x3dda3fca +543, 0x229e80a4 +544, 0x3d94a4b7 +545, 0x5d3d576a +546, 0xad7818a0 +547, 0xce23b03a +548, 0x7aa2079c +549, 0x9a6be555 +550, 0x83f3b34a +551, 0x1848f9d9 +552, 0xd8fefc1c +553, 0x48e6ce48 +554, 0x52e55750 +555, 0xf41a71cf +556, 0xba08e259 +557, 0xfaf06a15 +558, 0xeaaac0fb +559, 0x34f90098 +560, 0xb1dfffbb +561, 0x718daec2 +562, 0xab4dda21 +563, 0xd27cc1ee +564, 0x4aafbc4c +565, 0x356dfb4f +566, 0x83fcdfd6 +567, 0x8f0bcde0 +568, 0x4363f844 +569, 0xadc0f4d5 +570, 0x3bde994e +571, 0x3884d452 +572, 0x21876b4a +573, 0x9c985398 +574, 0xca55a226 +575, 0x3a88c583 +576, 0x916dc33c +577, 0x8f67d1d7 +578, 0x3b26a667 +579, 0xe4ddeb4b +580, 0x1a9d8c33 +581, 0x81c9b74f +582, 0x9ed1e9df +583, 0x6e61aecf +584, 0x95e95a5d +585, 0x68864ff5 +586, 0xb8fa5b9 +587, 0x72b1b3de +588, 0x5e18a86b +589, 0xd7f2337d +590, 0xd70e0925 +591, 0xb573a4c1 +592, 0xc77b3f8a +593, 0x389b20de +594, 0x16cf6afb +595, 0xa39bd275 +596, 0xf491cf01 +597, 0x6f88a802 +598, 0x8510af05 +599, 0xe7cd549a +600, 0x8603179a +601, 0xef43f191 +602, 0xf9b64c60 +603, 0xb00254a7 +604, 0xd7c06a2d +605, 0x17e9380b +606, 0x529e727b +607, 0xaaa8fe0a +608, 0xfb64ff4c +609, 0xcd75af26 +610, 0xfb717c87 +611, 0xa0789899 +612, 0x10391ec9 +613, 0x7e9b40b3 +614, 0x18536554 +615, 0x728c05f7 +616, 0x787dca98 +617, 0xad948d1 +618, 0x44c18def +619, 0x3303f2ec +620, 0xa15acb5 +621, 0xb58d38f4 +622, 0xfe041ef8 +623, 0xd151a956 +624, 0x7b9168e8 +625, 0x5ebeca06 +626, 0x90fe95df +627, 0xf76875aa +628, 0xb2e0d664 +629, 0x2e3253b7 +630, 0x68e34469 +631, 0x1f0c2d89 +632, 0x13a34ac2 +633, 0x5ffeb841 +634, 0xe381e91c +635, 0xb8549a92 +636, 0x3f35cf1 +637, 0xda0f9dcb +638, 0xdd9828a6 +639, 0xe1428f29 +640, 0xf4db80b5 +641, 0xdac30af5 +642, 0x1af1dd17 +643, 0x9a540254 +644, 0xcab68a38 +645, 0x33560361 +646, 0x2fbf3886 +647, 0xbc785923 +648, 0xe081cd10 +649, 0x8e473356 +650, 0xd102c357 +651, 0xeea4fe48 +652, 0x248d3453 +653, 0x1da79ac +654, 0x815a65ff +655, 0x27693e76 +656, 0xb7d5af40 +657, 0x6d245d30 +658, 0x9e06fa8f +659, 0xb0570dcb +660, 0x469f0005 +661, 0x3e0ca132 +662, 0xd89bbf3 +663, 0xd61ccd47 +664, 0x6383878 +665, 0x62b5956 +666, 0x4dc83675 +667, 0x93fd8492 +668, 0x5a0091f5 +669, 0xc9f9bc3 +670, 0xa26e7778 +671, 0xeabf2d01 +672, 0xe612dc06 +673, 0x85d89ff9 +674, 0xd1763179 +675, 0xcb88947b +676, 0x9e8757a5 +677, 0xe100e85c +678, 0x904166eb +679, 0x4996243d +680, 0x4038e1cb +681, 0x2be2c63d +682, 0x77017e81 +683, 0x3b1f556b +684, 0x1c785c77 +685, 0x6869b8bd +686, 0xe1217ed4 +687, 0x4012ab2f +688, 0xc06c0d8e +689, 0x2122eb68 +690, 0xad1783fd +691, 0x5f0c80e3 +692, 0x828f7efa +693, 0x29328399 +694, 0xeadf1087 +695, 0x85dc0037 +696, 0x9691ef26 +697, 0xc0947a53 +698, 0x2a178d2a +699, 0x2a2c7e8f +700, 0x90378380 +701, 0xaad8d326 +702, 0x9cf1c3c8 +703, 0x84eccd44 +704, 0x79e61808 +705, 0x8b3f454e +706, 0x209e6e1 +707, 0x51f88378 +708, 0xc210226f +709, 0xd982adb5 +710, 0x55d44a31 +711, 0x9817d443 +712, 0xa328c626 +713, 0x13455966 +714, 0xb8f681d3 +715, 0x2a3c713b +716, 0xc186959b +717, 0x814a74b0 +718, 0xed7bc90 +719, 0xa88d3d6d +720, 0x88a9f561 +721, 0x73aa1c0a +722, 0xdfeff404 +723, 0xec037e4b +724, 0xa5c209f0 +725, 0xb3a223b4 +726, 0x24ce3709 +727, 0x3184c790 +728, 0xa1398c62 +729, 0x2f92034e +730, 0xbb37a79a +731, 0x605287b4 +732, 0x8faa772c +733, 0x6ce56c1d +734, 0xc035fb4c +735, 0x7cf5b316 +736, 0x6502645 +737, 0xa283d810 +738, 0x778bc2f1 +739, 0xfdf99313 +740, 0x1f513265 +741, 0xbd3837e2 +742, 0x9b84a9a +743, 0x2139ce91 +744, 0x61a8e890 +745, 0xf9ff12db +746, 0xb43d2ea7 +747, 0x88532e61 +748, 0x175a6655 +749, 0x7a6c4f72 +750, 0x6dafc1b7 +751, 0x449b1459 +752, 0x514f654f +753, 0x9a6731e2 +754, 0x8632da43 +755, 0xc81b0422 +756, 0x81fe9005 +757, 0x15b79618 +758, 0xb5fa629f +759, 0x987a474f +760, 0x1c74f54e +761, 0xf9743232 +762, 0xec4b55f +763, 0x87d761e5 +764, 0xd1ad78b7 +765, 0x453d9350 +766, 0xc7a7d85 +767, 0xb2576ff5 +768, 0xcdde49b7 +769, 0x8e1f763e +770, 0x1338583e +771, 0xfd65b9dc +772, 0x4f19c4f4 +773, 0x3a52d73d +774, 0xd3509c4c +775, 0xda24fe31 +776, 0xe2de56ba +777, 0x2db5e540 +778, 0x23172734 +779, 0x4db572f +780, 0xeb941718 +781, 0x84c2649a +782, 0x3b1e5b6a +783, 0x4c9c61b9 +784, 0x3bccd11 +785, 0xb4d7b78e +786, 0x48580ae5 +787, 0xd273ab68 +788, 0x25c11615 +789, 0x470b53f6 +790, 0x329c2068 +791, 0x1693721b +792, 0xf8c9aacf +793, 0x4c3d5693 +794, 0xd778284e +795, 0xae1cb24f +796, 0x3c11d1b3 +797, 0xddd2b0c0 +798, 0x90269fa7 +799, 0x5666e0a2 +800, 0xf9f195a4 +801, 0x61d78eb2 +802, 0xada5a7c0 +803, 0xaa272fbe +804, 0xba3bae2f +805, 0xd0b70fc2 +806, 0x529f32b +807, 0xda7a3e21 +808, 0x9a776a20 +809, 0xb21f9635 +810, 0xb3acc14e +811, 0xac55f56 +812, 0x29dccf41 +813, 0x32dabdb3 +814, 0xaa032f58 +815, 0xfa406af4 +816, 0xce3c415d +817, 0xb44fb4d9 +818, 0x32248d1c +819, 0x680c6440 +820, 0xae2337b +821, 0x294cb597 +822, 0x5bca48fe +823, 0xaef19f40 +824, 0xad60406 +825, 0x4781f090 +826, 0xfd691ffc +827, 0xb6568268 +828, 0xa56c72cb +829, 0xf8a9e0fc +830, 0x9af4fd02 +831, 0x2cd30932 +832, 0x776cefd7 +833, 0xe31f476e +834, 0x6d94a437 +835, 0xb3cab598 +836, 0xf582d13f +837, 0x3bf8759d +838, 0xc3777dc +839, 0x5e425ea8 +840, 0x1c7ff4ed +841, 0x1c2e97d1 +842, 0xc062d2b4 +843, 0x46dc80e0 +844, 0xbcdb47e6 +845, 0x32282fe0 +846, 0xaba89063 +847, 0x5e94e9bb +848, 0x3e667f78 +849, 0xea6eb21a +850, 0xe56e54e8 +851, 0xa0383510 +852, 0x6768fe2b +853, 0xb53ac3e0 +854, 0x779569a0 +855, 0xeca83c6a +856, 0x24db4d2d +857, 0x4585f696 +858, 0xf84748b2 +859, 0xf6a4dd5b +860, 0x31fb524d +861, 0x67ab39fe +862, 0x5882a899 +863, 0x9a05fcf6 +864, 0x712b5674 +865, 0xe8c6958f +866, 0x4b448bb3 +867, 0x530b9abf +868, 0xb491f491 +869, 0x98352c62 +870, 0x2d0a50e3 +871, 0xeb4384da +872, 0x36246f07 +873, 0xcbc5c1a +874, 0xae24031d +875, 0x44d11ed6 +876, 0xf07f1608 +877, 0xf296aadd +878, 0x3bcfe3be +879, 0x8fa1e7df +880, 0xfd317a6e +881, 0xe4975c44 +882, 0x15205892 +883, 0xa762d4df +884, 0xf1167365 +885, 0x6811cc00 +886, 0x8315f23 +887, 0xe045b4b1 +888, 0xa8496414 +889, 0xbed313ae +890, 0xcdae3ddb +891, 0xa9c22c9 +892, 0x275fab1a +893, 0xedd65fa +894, 0x4c188229 +895, 0x63a83e58 +896, 0x18aa9207 +897, 0xa41f2e78 +898, 0xd9f63653 +899, 0xbe2be73b +900, 0xa3364d39 +901, 0x896d5428 +902, 0xc737539e +903, 0x745a78c6 +904, 0xf0b2b042 +905, 0x510773b4 +906, 0x92ad8e37 +907, 0x27f2f8c4 +908, 0x23704cc8 +909, 0x3d95a77f +910, 0xf08587a4 +911, 0xbd696a25 +912, 0x948924f3 +913, 0x8cddb634 +914, 0xcd2a4910 +915, 0x8e0e300e +916, 0x83815a9b +917, 0x67383510 +918, 0x3c18f0d0 +919, 0xc7a7bccc +920, 0x7cc2d3a2 +921, 0x52eb2eeb +922, 0xe4a257e5 +923, 0xec76160e +924, 0x63f9ad68 +925, 0x36d0bbbf +926, 0x957bc4e4 +927, 0xc9ed90ff +928, 0x4cb6059d +929, 0x2f86eca1 +930, 0x3e3665a3 +931, 0x9b7eb6f4 +932, 0x492e7e18 +933, 0xa098aa51 +934, 0x7eb568b2 +935, 0x3fd639ba +936, 0x7bebcf1 +937, 0x99c844ad +938, 0x43cb5ec7 +939, 0x8dfbbef5 +940, 0x5be413ff +941, 0xd93b976d +942, 0xc1c7a86d +943, 0x1f0e93d0 +944, 0x498204a2 +945, 0xe8fe832a +946, 0x2236bd7 +947, 0x89953769 +948, 0x2acc3491 +949, 0x2c4f22c6 +950, 0xd7996277 +951, 0x3bcdc349 +952, 0xfc286630 +953, 0x5f8909fd +954, 0x242677c0 +955, 0x4cb34104 +956, 0xa6ff8100 +957, 0x39ea47ec +958, 0x9bd54140 +959, 0x7502ffe8 +960, 0x7ebef8ae +961, 0x1ed8abe4 +962, 0xfaba8450 +963, 0xc197b65f +964, 0x19431455 +965, 0xe229c176 +966, 0xeb2967da +967, 0xe0c5dc05 +968, 0xa84e3227 +969, 0x10dd9e0f +970, 0xbdb70b02 +971, 0xce24808a +972, 0x423edab8 +973, 0x194caf71 +974, 0x144f150d +975, 0xf811c2d2 +976, 0xc224ee85 +977, 0x2b217a5b +978, 0xf78a5a79 +979, 0x6554a4b1 +980, 0x769582df +981, 0xf4b2cf93 +982, 0x89648483 +983, 0xb3283a3e +984, 0x82b895db +985, 0x79388ef0 +986, 0x54bc42a6 +987, 0xc4dd39d9 +988, 0x45b33b7d +989, 0x8703b2c1 +990, 0x1cc94806 +991, 0xe0f43e49 +992, 0xcaa7b6bc +993, 0x4f88e9af +994, 0x1477cce5 +995, 0x347dd115 +996, 0x36e335fa +997, 0xb93c9a31 +998, 0xaac3a175 +999, 0x68a19647 diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/data/pcg64-testset-2.csv b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/data/pcg64-testset-2.csv new file mode 100644 index 0000000000000000000000000000000000000000..7c13e3172d0e7798200f58cde9362ad0bd4a22e1 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/data/pcg64-testset-2.csv @@ -0,0 +1,1001 @@ +seed, 0x0 +0, 0xa30febcfd9c2825f +1, 0x4510bdf882d9d721 +2, 0xa7d3da94ecde8b8 +3, 0x43b27b61342f01d +4, 0xd0327a782cde513b +5, 0xe9aa5979a6401c4e +6, 0x9b4c7b7180edb27f +7, 0xbac0495ff8829a45 +8, 0x8b2b01e7a1dc7fbf +9, 0xef60e8078f56bfed +10, 0xd0dbc74d4700374c +11, 0xb37868abbe90b0 +12, 0xdb7ed8bf64e6f5f0 +13, 0x89910738de7951f +14, 0xbacab307c3cfd379 +15, 0x2cf7c449d8b927a6 +16, 0xdcf94b3a16db7f0e +17, 0x8a9d33d905a8792e +18, 0x4cb9eb2014951238 +19, 0x6c353acf7b26d6f1 +20, 0x73ff53d673aa30c +21, 0x1fd10760015eca68 +22, 0xabae0aa9021eeba8 +23, 0xa5ae363a868ee2bb +24, 0x9d89e0f041de6631 +25, 0x6238b133c3991a65 +26, 0xff49267d75fef51a +27, 0xfb180656ce13c53f +28, 0xaf7fadf36128712d +29, 0xa6847fc6f339c63e +30, 0xb03e0b80d71ea5bc +31, 0x63905abcb43969af +32, 0x2295af3ee00a3bba +33, 0xb8b375b994330415 +34, 0x867d9ef1d8716a3b +35, 0x4f6c02f5601b4e18 +36, 0x7c5fb4c16c470d18 +37, 0xe3b57986b804b343 +38, 0xef1d79d212aca692 +39, 0x5b98774c8806209c +40, 0x924fc76bac38a5d1 +41, 0x5266084c412ddeed +42, 0x98240bf9b831d6a3 +43, 0x5681599e81219442 +44, 0x6441248fc2ba92bc +45, 0xe3e9051a540349ea +46, 0x3a2700034390baa3 +47, 0x9f893155b6d402bc +48, 0x158207910c6d8aef +49, 0xd5282ab7608c2cbc +50, 0xc97f4651669dee4f +51, 0x3d4750d95103ed60 +52, 0xe0614542caac1f04 +53, 0xefe5092144cfc6c +54, 0x560bc486abd7e9ae +55, 0x2678b71392daa4b8 +56, 0x734970d3dc2ba416 +57, 0xcbdbe849e51e4aaf +58, 0x3b0b5e28b491556c +59, 0xd51449ac45abd88 +60, 0x6790b59991f1b7ab +61, 0x32d1c039ff2415bc +62, 0x173b9772f24f72e0 +63, 0x9490a9ca9f883b1b +64, 0x4c775989e6214222 +65, 0xac07db37e6ee6114 +66, 0x331371b2e3f10aee +67, 0xf12e5326c21c28e4 +68, 0x5d77dc280c70d614 +69, 0x1b01bd17a2f281ec +70, 0xa10d3b5882938487 +71, 0xed5a0033c394ae8f +72, 0x70bc8ea568ea44b4 +73, 0xf4600ae77965e730 +74, 0x7ff92c0b321ce233 +75, 0x6cdbc87d0cc1d670 +76, 0x9ec64f0cf2000eb1 +77, 0xfebea50259800f68 +78, 0xf2edf9019a8fd343 +79, 0x75c584ac042e5468 +80, 0xc1fa8481d5bf9a1d +81, 0x7f57180168514ac2 +82, 0x878100716b94f81e +83, 0xc929406e3af17fd2 +84, 0x6a26e2c013e4bf4d +85, 0xbc071d8848280955 +86, 0xb60d75abbfd1bdac +87, 0xee9b76afeca9fa69 +88, 0x1d6c399d2f452810 +89, 0xbaa0bc1621e25c83 +90, 0xed6ba792f8671ba5 +91, 0xf7ca02c2ab11d8d7 +92, 0x3c3cadadf0b21e3 +93, 0xdd1784571e864e9c +94, 0xfb2f992015157509 +95, 0xf50bb9f0d3ced743 +96, 0x261565f75c3e185f +97, 0xf8fe33b284513e60 +98, 0xe3d2d10b5e024664 +99, 0xd28717566242cf35 +100, 0x7ae07d133ac5b789 +101, 0x3b7ccaaa53ac338e +102, 0xcd480bace4871650 +103, 0xec6c78f923c080e9 +104, 0x44211d0ff8919d59 +105, 0x89f79af76d2a45fe +106, 0x71583fd8a837548b +107, 0xee57269c261511f5 +108, 0xa5ee8f3b128c5d1 +109, 0xbb64c20ed0765a17 +110, 0x9d4790ab2eeaf7e4 +111, 0x742f3db806d9e98 +112, 0xb81ec97aed6a0d1b +113, 0x41808b34f6a8a23 +114, 0xc20913af175dfd4d +115, 0x834427db263b22bb +116, 0xedd9c632e611828a +117, 0x10eac8524496f571 +118, 0xd76091b97eb00ab7 +119, 0x111298ae9fe95666 +120, 0x5824b2e2a6719c43 +121, 0x6e280ec539e934ed +122, 0xf74fd832df90083e +123, 0x8fee6d0f241c2e97 +124, 0x4244f331c2f19c3c +125, 0x3dde75a845cce97f +126, 0xe35bb8e635a9915b +127, 0x39d2943037f7932e +128, 0x1fe2d134201d0970 +129, 0x49d00b63c749b804 +130, 0x960c2942cd4e4e04 +131, 0x8dd8e009dbc0435f +132, 0xcf493495c3a055cd +133, 0x8f7b5a1c0f9fe9cd +134, 0x49d5f90374641a25 +135, 0x69b3932073d3524c +136, 0xd170603e7de84ee2 +137, 0xa062ba3ed3539948 +138, 0xf5861cc5b5d56c82 +139, 0x5e914998a30c7e76 +140, 0x8d77f2ad1503c0f1 +141, 0x980b6a9e3b4181fb +142, 0xd9299cd50694c084 +143, 0x253dc0f8f1cec4c5 +144, 0x68110fb9d1b3e695 +145, 0xe8f3120d0aabc461 +146, 0xb066e7df0dfb042 +147, 0xd29ce0f797e6b60b +148, 0x6a569bb7ca33bd42 +149, 0xd46e08b2dc2385f8 +150, 0x28c61d11d055767 +151, 0x5d73aa3d1a2bb725 +152, 0x1421191e1c14829a +153, 0xa711bfb6423df35e +154, 0x461af97a86308006 +155, 0xb3e1018ff3519367 +156, 0xf19cf866a268ef2b +157, 0x207715eac9199d1d +158, 0xdd621c410975b78c +159, 0xf390aea68683610 +160, 0x617a2d107a0047d9 +161, 0x6e05ac416e5bebf0 +162, 0x7d253e70506c1bed +163, 0xf9f96f4a7dd53810 +164, 0xc693b29cb1573f73 +165, 0x4f1146b0020ea544 +166, 0x45140608fbd40579 +167, 0xdcf57219828ce6be +168, 0xe19d58cca37b5b32 +169, 0x82bda95b2a161235 +170, 0x5823c3d8a2b6c9ba +171, 0xfeb2e74092fdf89a +172, 0x50e1ad1abc8f869d +173, 0x2ec63d0c105eb8da +174, 0xe14e1c4845a3264a +175, 0xcff53670455eb6aa +176, 0xaafaccd24619fa3e +177, 0xf55a988486e2422a +178, 0xecfba16a90ff4d04 +179, 0xbf8d36c2f644757a +180, 0xdc56ed75a0dd6249 +181, 0x3f45023eff17c3bb +182, 0x2428bbfe90023fab +183, 0xab892c611adcb70c +184, 0xb6f13d8c0c2b9d74 +185, 0x2ac3fb11d224f2a8 +186, 0x65433dcfae2d9351 +187, 0xe906859ae4b45f82 +188, 0x8fb7f5f093d76a3b +189, 0x940dd290b5e88d1a +190, 0x31b27d21bef116e7 +191, 0x86a964e2c83b5296 +192, 0x85ffd17bc079a9e8 +193, 0x16c47c724e7ab7f1 +194, 0xfb6098a9867e7d7f +195, 0x9246fb69092c6cb2 +196, 0x1a4033572760f32 +197, 0xc5cc568a8b273b84 +198, 0xfa6f9f2fbdd44abc +199, 0x9701b8e087718ba3 +200, 0x51d6a7dcf73f8f3a +201, 0x30008172cc6a972d +202, 0xac2ab49a5ca6ac81 +203, 0x31f28ef79461e54c +204, 0x93e35a8da8cc6132 +205, 0x9a2c58beeba3d5b9 +206, 0xf6615c1de266ac39 +207, 0x127ff9f8166b766b +208, 0x7ffe380e80a69556 +209, 0xbe7d2c228e1542f7 +210, 0x2d5ebb4e50ba1746 +211, 0x63585761ae1bf684 +212, 0x1019eb5cee022fea +213, 0xb9d3540ab58da30d +214, 0x1677f4cb45620eb9 +215, 0x6524baee51783822 +216, 0xdf9f2ddcfabb0adc +217, 0x78e8acc43b287935 +218, 0xe9a1974e999222b5 +219, 0xc41324ec2291e780 +220, 0xea52abc9ecdcbc9f +221, 0x209d7bcd46ec6b04 +222, 0x12d504c09803db2e +223, 0x1200e6bf21475d81 +224, 0xde6d3c2b35fd2cfc +225, 0xa2526900ac33bd3c +226, 0x7f1f5290fc432bc5 +227, 0x29ddfb380a3d69c8 +228, 0xac79cb6942a2909d +229, 0x516996685b67a92a +230, 0xb5fc39041cb828bb +231, 0x75d9d8ca0644a276 +232, 0x81e98b76be92a3e9 +233, 0xca27888fafe12179 +234, 0x17be2ae039925765 +235, 0x9429846c0e6d0342 +236, 0x327dfd50439815e9 +237, 0xcee20cd7bc254aeb +238, 0x7d250389f453f29e +239, 0xfd1b232a85c95569 +240, 0x2ed55fac80f3e9e9 +241, 0xf6886c20417a1be7 +242, 0xcd08e61f0b0fdfde +243, 0x7b33e34da5c27bff +244, 0xd043c4b7d5603dd5 +245, 0x9a544e4c70a3b686 +246, 0xa7b60398c381f771 +247, 0xe9e7a3487c4bd4f2 +248, 0x10b58fdfe1ff112c +249, 0xd5c1c9748c0f4ceb +250, 0x61be9d09159d54ff +251, 0x5356f51e8239f510 +252, 0xfe7889d9b202ecef +253, 0xc7fc19ca5d263d5d +254, 0x7c4c07e61dfd9f69 +255, 0x6c315fe5015f300a +256, 0xe0a5bc00039747b4 +257, 0x16397fdcf829ee80 +258, 0xb55aee80d16a5169 +259, 0xca0609944d007eea +260, 0xcc982249f65a02ce +261, 0x528161feb149c148 +262, 0xcbf08ba49b41c006 +263, 0x39af1ff0b6f14138 +264, 0x5cc036be69799aec +265, 0x6adde125b1db21c5 +266, 0x8a99d83d6b613b67 +267, 0x1cd43fca9451f74c +268, 0x682dbb26ecc96365 +269, 0x13b4be2ceb43e3 +270, 0xbe8fbc3b6f4f581e +271, 0xda148a2f4bda5719 +272, 0x239106ca3319f393 +273, 0xb42b4dde641f0dd5 +274, 0xd233cfdf4cb0af74 +275, 0xfb5919d905589afc +276, 0xd802a8860c10b66a +277, 0x6c923e1d00e7b5bc +278, 0xfacce1134f383b89 +279, 0xf9570abda7a6d553 +280, 0x80f0f9796a208f18 +281, 0xc0e1df5280951c57 +282, 0xe9f143f08257bbe0 +283, 0x79e4c6463123d588 +284, 0xdd2118583f2b1684 +285, 0xb399ff5f2329fa18 +286, 0x4b3e9ebae96f813c +287, 0xc484dbf247787384 +288, 0x921865eb97603f2c +289, 0x18063c68e257d300 +290, 0x643181f345e7fc26 +291, 0x12e0b0e8eadf9fa7 +292, 0x79e613fe73dfa354 +293, 0x6db4c59203b7217a +294, 0x6c7a0e9ba6139eaf +295, 0x9617c7ac4e3f6d97 +296, 0x1f68a7b4fb1b4b75 +297, 0xef0b7ab24944f466 +298, 0xaf1dee1f4be1bc89 +299, 0xd2e355c959f5fd8d +300, 0xe594c3fb95d96efc +301, 0x9554766ca3342906 +302, 0xa4bbdc77d12842c +303, 0xb62400211ee489a8 +304, 0x91abadaaa3bbe67c +305, 0xd371eeb91deb42bb +306, 0x883bab35cbd2b6e5 +307, 0xd030c3d9411a9041 +308, 0xff3c110a858ff000 +309, 0x59bdf5ca47d0bde7 +310, 0x2bc80fa3cdba1853 +311, 0x6444ccb652662cb8 +312, 0xc0c7e256b9e90339 +313, 0x70714ea9c9d72302 +314, 0x96a0142f9d897d27 +315, 0x209a9097c5a91ef7 +316, 0xb9e33afc5171e009 +317, 0x47b37af433a58d40 +318, 0x30cc4ffbfa831d26 +319, 0xdcea4a85ff815466 +320, 0x907d5bd027f2e5cc +321, 0x7c081f6852e04a4b +322, 0xe61950749c1d502b +323, 0x1604e937ee69834a +324, 0xb2372d952dd25309 +325, 0x53f6a5b834c72577 +326, 0x2ce7a74395e0b694 +327, 0xacbf9ab4fe91f225 +328, 0x5ce1e63d3a2bb90f +329, 0x54740da3a5ed139b +330, 0xf194ddb39f29880b +331, 0x3305374f5d8ec08b +332, 0x831dd0164927ff4a +333, 0x625baa78e4458cf +334, 0x29d27dc0a4a71152 +335, 0xe227bae9a1401034 +336, 0xca0c209831846b2b +337, 0x8e8cc54b08b5a411 +338, 0x38f2b4acaac27db6 +339, 0x8ec88baac814e86b +340, 0x31c08e46b007bde +341, 0xb686c02722794c09 +342, 0xb77cf8fc682e3907 +343, 0xa56334e7f606f4b2 +344, 0x9c80b127bddd5f4f +345, 0x12df14834cd858bf +346, 0x3f14762a9cf5fb9f +347, 0x930a70941ef5779e +348, 0x64e96c849c30c080 +349, 0xfdf53bfba1300484 +350, 0xec7a9363c21bc616 +351, 0x26e9fd6a115ecb47 +352, 0x9707a84b5bc77fbb +353, 0xb23b2737b20d5903 +354, 0x22f4825ae80f6501 +355, 0x500644b12be6a01b +356, 0xb746645b2af082db +357, 0xe6af051f697892f8 +358, 0x577c724248a1cfc6 +359, 0x3d2b6a434c84eed3 +360, 0xd260f5efd7328314 +361, 0x95c16cc84bb3f55c +362, 0x7a01b2e4e0e80ca7 +363, 0x41930c3ce70a0935 +364, 0x1299bccf39d4e110 +365, 0x494883ba1a8a87f +366, 0x9478ecfe2d918e60 +367, 0x30ec9a5670cda8af +368, 0xf9bc877e833e2b99 +369, 0x1b83a0acfbb4a8db +370, 0x73bc1740c0d18880 +371, 0x65086ca9773cb3e1 +372, 0x3b78c3ccd63cff2e +373, 0xbfae748795acfb31 +374, 0xa4c9d5d56a15ba20 +375, 0xb9cb41721e52b71e +376, 0x1532f15d4dc47748 +377, 0x5a4d647a4b9ee632 +378, 0x8513c7c5a50898d9 +379, 0x6d3d98ccd5461b2e +380, 0xa65e99be2fe98d6 +381, 0x31abc8855334a0e5 +382, 0xf1ed22a661dca5b8 +383, 0x299e2b63229e03be +384, 0xda201a06687bce48 +385, 0xd27794b302142c55 +386, 0x642bd3e1c7898a9d +387, 0x777f1ff00afa1a87 +388, 0xd2f1c84fb3877baa +389, 0xae417583289191fd +390, 0xd641f1d88e0e2d55 +391, 0xc1f1d98fb5d18ebf +392, 0xb0f72aecdadce97b +393, 0xe9b8abc764f6018a +394, 0xd2a37cff8e890594 +395, 0x2dd70d631a528771 +396, 0xbf8ba0478c18e336 +397, 0x1630bf47f372ce0a +398, 0x6d04ea20dc3f46b8 +399, 0x6591881bf34337f2 +400, 0x33c149c7eb5b4103 +401, 0xf01a8c9857c86748 +402, 0x184348cdfc16d215 +403, 0x141168b253d2ed7 +404, 0x52aaf012ef50a6f1 +405, 0xfda1722387e16f4c +406, 0x43c30f57d6c038fa +407, 0xd4a8611f5f96d214 +408, 0x2c512ce17e987f2c +409, 0x961ce450f0fa2822 +410, 0xf55a506ec6cea9cd +411, 0xb76d694d9c7f5ef6 +412, 0xfb029216dbd8e988 +413, 0x93162501896a0081 +414, 0xfbbbd2c5ab300f5c +415, 0xd648b6da7387d491 +416, 0xc73b4697471d9d98 +417, 0xe37412bf1c93ee76 +418, 0xa1a96d96570e6637 +419, 0x5b3ab4f82428f65c +420, 0x873d849b188aa36f +421, 0x39fbee0ffc9fa9ff +422, 0xc70d21b744d677fe +423, 0x2b8a43c23043d209 +424, 0x93c33eaa37370d16 +425, 0x8930ac1880f2b0ef +426, 0xac01d27707036af0 +427, 0xc2af3fee504343a0 +428, 0x1c1dae2ad5535d97 +429, 0x9ffc21804b76a480 +430, 0x69f903412cc13563 +431, 0x9d3c4e2759a0c47d +432, 0xb1a8f894be6302b9 +433, 0x95e1fd7951479506 +434, 0xbb9e6c03cd4ae8e3 +435, 0x85206010c9b737cf +436, 0x767e813694d6238c +437, 0x4969af329ccbb30a +438, 0x3aa9af1075aaea5c +439, 0xb1ff519e8118a993 +440, 0xb21a23a3c91180fe +441, 0x320b24582ca3fd88 +442, 0xf8ca56415fb4e453 +443, 0xabd0899c07205e77 +444, 0x87fdc7a44b4ad50f +445, 0xd75744911641a278 +446, 0x7c8c9a65df6fcb95 +447, 0x79d785e3c7a5b695 +448, 0x421e4565ba1f592f +449, 0x27f87eb2517835cf +450, 0xb62cc4297441c83e +451, 0xd817a80ac815ca6d +452, 0xad84388130df2aa8 +453, 0x5e6b1640452d6ac8 +454, 0x936285e15edce2a3 +455, 0x903bccc4969768e8 +456, 0xefc2cb7b109d3140 +457, 0x633e9dfdda2d903a +458, 0x2a2f3225925678a1 +459, 0xe07eac91a27f8547 +460, 0xe50ced40eda78cb3 +461, 0xc5b22500e1c7441 +462, 0x32becf61bca3aa72 +463, 0xa2e37c4b30671344 +464, 0xc9f1c1910f45d544 +465, 0x9b50333b2dcdf730 +466, 0x310bfd53a1684b94 +467, 0x1e1dc21e66ac6455 +468, 0x81876c2bfb1ed5a1 +469, 0xd0c54a3e25eadc7b +470, 0x3791b6fbbd5c7ba0 +471, 0x133be57356c599fc +472, 0x8d1148eb8e83fdea +473, 0x311aedba0d8b42cc +474, 0x1142ae52745f94bb +475, 0xc5f4ab2fbde8c4a3 +476, 0xd23be827b5b24f6d +477, 0x65f95194cd122715 +478, 0x4b48969d73125922 +479, 0x46f165052b8ff988 +480, 0x5c689f94b9275ff4 +481, 0x93b03823ff2d536b +482, 0x871f3775aa4e3523 +483, 0x5af829f7cc0f66a5 +484, 0xa32e05739cbeac8c +485, 0xacff1856ddace0fe +486, 0x8eeb5e7f991a5322 +487, 0x6325c2720e0dbdea +488, 0x9fb817bc4fdf5200 +489, 0x9786f0d850e43d78 +490, 0x571f76dd7f9fb77a +491, 0x4d9e94e181cbc63f +492, 0x8bb632d3376c547a +493, 0x9cc26d9efd1c88b9 +494, 0x9c5d49579df52b0b +495, 0x6201abf7e1cda07b +496, 0x90d68f0c6c884963 +497, 0xfc5b66188ef7f561 +498, 0x6d9303cf2e0e0f95 +499, 0xd7cfcff535f5ed07 +500, 0x14d1a1228daa4ac6 +501, 0xe00ef5762f66ae50 +502, 0xf113a79471582978 +503, 0x430985281785dc7a +504, 0x31914108c206ed5 +505, 0x7ba6707b6419971c +506, 0x2ec63b033ce112e5 +507, 0xf8bcd36ced3b41e3 +508, 0xe5cf908c8010414b +509, 0xf5ee224b7c703e30 +510, 0x9a9733af0b12338b +511, 0x83e18cc00ace34f8 +512, 0xd52cff39e23008b8 +513, 0xa700578136b9c0c5 +514, 0x3fa179d32ac51f99 +515, 0xef2d5eab6d4ad380 +516, 0x709024a5abd032df +517, 0xc607c7ee349ede87 +518, 0x803d784e9731eb5f +519, 0x2ef06f4ba769282d +520, 0x4bc1dca1e9f07eb9 +521, 0x930c958a7a72f94d +522, 0x249bc8db2cc7a3bf +523, 0x3845305798f9a5d +524, 0x6f137eca9ab6f948 +525, 0xc31f5a963d31bd67 +526, 0x9d39693d5383626f +527, 0x52fb41c335a8b98e +528, 0xb79d1a29a06006ec +529, 0x7c0926a7a3eda2cc +530, 0xffdf5214406fd53e +531, 0xc6aa02a7e94282b9 +532, 0xd4a4431b4aa301ee +533, 0x4271cc0f9420d3ab +534, 0x26fccd7cc7fc2485 +535, 0x330594bb945b8d5a +536, 0x6ea8eaad12e5cb8c +537, 0x831c3467726bede3 +538, 0x31d1eb10017eaa61 +539, 0xc7aa75e41508f5cb +540, 0xde51810f0cadd0b5 +541, 0x50e5b3e73692f80b +542, 0x82107ec55636e188 +543, 0x9828ef175d843ab4 +544, 0xb8edc6a860dd421e +545, 0x25c0c138fd537ac3 +546, 0x47e72a771e8eb563 +547, 0xbb0f8c5333f4a2cc +548, 0x91750d2fb9b2d479 +549, 0xe662d8f6fe38df36 +550, 0x72a6d879fb5619f0 +551, 0x6817c7878dcbf077 +552, 0x4e7741cb484661e8 +553, 0x3b3b3ba0be5711bf +554, 0xa6989f5d25868765 +555, 0x43c276398997e4e0 +556, 0xdcbe16a94da28870 +557, 0x454936980a699c99 +558, 0xac614bfa8f0266c6 +559, 0x9174841392e213d5 +560, 0xa0e2acffc5fc9d1f +561, 0xe53a08a7a0e6521a +562, 0x2b845cf7c24172e0 +563, 0x265a4fc5f7adec0d +564, 0x1f34fbe5f1e49420 +565, 0x139181f6fb647f20 +566, 0x88c35d46e2fcd05e +567, 0x2a6d5b55903c0459 +568, 0xcea28eb621ad7bf1 +569, 0x5c9cdc13e7aaa30 +570, 0x5fe63e14746e7103 +571, 0x7923e53d73835db9 +572, 0x376e661210bf1b06 +573, 0x5b1cab85450efdd5 +574, 0x3908dc096c70b452 +575, 0x4825e303cd1f396f +576, 0xed476bfd702957c3 +577, 0x6acc013aff5db743 +578, 0x62c80b776343d488 +579, 0x9c75edcd5b012697 +580, 0xaa053362a3b9770a +581, 0xa907e236c7c07e94 +582, 0x15b2c380451692c0 +583, 0x94f79142697bd61f +584, 0xbc657d31ea98d44f +585, 0xcbaa5e52517a1f5e +586, 0x96aa2e44a7c4a03f +587, 0x216d3c66db2b515d +588, 0x157001807e3ca88a +589, 0x52b3a596bdd3859a +590, 0xed747e7fc5e3adac +591, 0x78fd765ddb2c448d +592, 0xe53dc7299ed8614e +593, 0x75ad41fb1d7a790a +594, 0xc14f6b944b0e6cb1 +595, 0x7c314b69fce3df1c +596, 0xb56d82eb740d7abc +597, 0x5132a93c41251fdb +598, 0xe3ce35bd2a82f958 +599, 0x440571a981c722f2 +600, 0x194cdfd9f186bc9 +601, 0xb89e522a5db00939 +602, 0xad35f339f68df3c8 +603, 0xa82ab18420322293 +604, 0xaffa6df9b72b27c4 +605, 0x9615694d23beaa2c +606, 0x1d82ebe563abad91 +607, 0xab50ef65fbd94385 +608, 0x1b070dbd70a9a14 +609, 0x2ececa796abbadf0 +610, 0x6bbeafe9e81ab2a2 +611, 0x60dcd0d2a9b76914 +612, 0x1e748039ef05c33f +613, 0x6d4d17f2213ccdff +614, 0x9fa56132957bc987 +615, 0x60a17185de2428eb +616, 0xb56038ddf306479c +617, 0x3b1db5df92d06d8b +618, 0x24d1bba8bdedf580 +619, 0xbfb7e6740ebaa4d9 +620, 0xab31c4473e46f61d +621, 0x6deb3cdd8fd5869f +622, 0x23032e47746d72d6 +623, 0xa9e72d734e10f2e8 +624, 0xbffd199b6157bc23 +625, 0x29f8254df273fb62 +626, 0xb076142130ee55ec +627, 0x5b0b08374126c309 +628, 0xea4536aae979521f +629, 0xc064e7abec91a174 +630, 0x46133ef80c59d935 +631, 0xf0227e2da1b14160 +632, 0x675a76641e1af5a +633, 0x2f50a069b33d198c +634, 0x3ded5a65e1d657eb +635, 0xbb6999b020694f6b +636, 0x86b2f2b33487aed7 +637, 0x76e14e85f8bfb4cf +638, 0x38f7f1e44bd4e0db +639, 0xc1a7d41b7e80d4ae +640, 0x1dfaaf80bbceb42e +641, 0x3f51c11497720c2b +642, 0xce6da1415ddb8b80 +643, 0x7377d8bcd359b5f3 +644, 0xe077208f3f810aca +645, 0x9a06a8a2dacbffce +646, 0xca1f99156b09b735 +647, 0x2ff9a93064d91451 +648, 0x50f3ea93f351a7ef +649, 0x606fceccb07054de +650, 0x7e83d6d2f8f6685d +651, 0x78f3995291c5d407 +652, 0xd28d2460e22d0228 +653, 0x2c5636f68a0054dd +654, 0xd9fafb1c56c8f6cb +655, 0xe39889b5f9d74464 +656, 0x1355372bf5db2cc1 +657, 0x26768426b9ac323 +658, 0x4af1dbdc1111fd89 +659, 0x66973587943b927f +660, 0xf86f5f50684dfb1d +661, 0x1247d574ff79b534 +662, 0xc8039f3259210fe2 +663, 0x79b573235c92a9f5 +664, 0x213f642d8450e2f0 +665, 0x5db7706973376566 +666, 0x6182c12e69b373d7 +667, 0x3e5ac47300aec07f +668, 0x4b5b6c57b1574376 +669, 0x6b7fcceefd56b17c +670, 0xf656c3455cb9d4b8 +671, 0x7577e2e13329721f +672, 0xf33c0c53ce956e8d +673, 0x7d0f328ee356174 +674, 0x10ec9a168088686e +675, 0x71ef1776d062dfa +676, 0xaa7b590a488a6bc4 +677, 0x38612b6dd8049a1c +678, 0x939045e36874f731 +679, 0xcb9d1d74c56d5ac9 +680, 0x54f1c1c8fef1d8ff +681, 0x3ee4b85c8c7e939e +682, 0xb9b4608e019f352c +683, 0x79d4701275d12e6a +684, 0x2632a2d9835c7f19 +685, 0x1662cd9fba293692 +686, 0xbcb70265115ee944 +687, 0xdc43fb9761468604 +688, 0xe3eec4e7d3871352 +689, 0x829531753226989d +690, 0x2748cc67f540e074 +691, 0x39c4af25d607837d +692, 0x741a243f4cb5df99 +693, 0xda1353287e18b49a +694, 0xa6735689d751ea74 +695, 0x46326d587340ce0b +696, 0xc18531df4550012b +697, 0x6f7901e05dd4b818 +698, 0xfb966afc4c001d63 +699, 0x6dc10fca67a9cfdb +700, 0xd6527ffadf0feaae +701, 0x3b900172045e25d +702, 0xb7dd594cdded6a46 +703, 0x6602aee7ec1599fc +704, 0x7fbf12f23747546a +705, 0x32e63f662bd2de0d +706, 0xedf47770b67ed641 +707, 0x331bef83481c5c2a +708, 0x8fc4256fdf05158c +709, 0x98eba48dabccf5e0 +710, 0xdbc2f2cdb7b1c154 +711, 0x7777755616517ad3 +712, 0xd473c147d2628ac1 +713, 0x861e15d1d760b5a7 +714, 0xf4d25926405ecb07 +715, 0xb7739c69effff86e +716, 0xe97fbafa6f96830c +717, 0xf13e8a334e8bede1 +718, 0xcd60010cba4ee4f9 +719, 0x1f537ac2b82e6008 +720, 0x1fda8d781a89140a +721, 0x9dc204f3f4a463f0 +722, 0x456dcd18eb56a1ab +723, 0x629957bc87bd16a1 +724, 0x2c8000ddb8c75253 +725, 0xc31dae9ec8449284 +726, 0xdac05c8baa2b691a +727, 0x21ff7be9ffa3e7ac +728, 0x844f4b5ed4ee08d0 +729, 0x651f913fd636c994 +730, 0xca3e71a2110b2d49 +731, 0x7709bc42253ed09d +732, 0xbb164d45b6569d43 +733, 0x90ec2f040c20a112 +734, 0xfa6e77e9166f5be4 +735, 0x6b6d12c1842d587d +736, 0xfcd7ff8466e25e2a +737, 0x6a5a2ed8bd971297 +738, 0x2ec35f6bba5adcbc +739, 0xc83676e16651249a +740, 0x458f6064cefe10ba +741, 0x90d54d527e6cd028 +742, 0xa5613e88db27c388 +743, 0x331e0c7d85aa1abc +744, 0x8cee4977e210358 +745, 0xfcae379aa6cbff8e +746, 0xd1407afc97a57e86 +747, 0x1fab25c864f094ae +748, 0xd914864a63004552 +749, 0x4214d226a20f1384 +750, 0x3f4e0d80c488b715 +751, 0xc5ca2f654024b7c8 +752, 0xc1e27a124e7c821c +753, 0xd890a915ffc7918c +754, 0x22fba040ce51a9f8 +755, 0xbf61cebd8891617a +756, 0x7846609ee228e319 +757, 0x536d1854375509b8 +758, 0xbbfb45fc6e666f50 +759, 0xd85b4c0527f9d7d6 +760, 0x528cc9c7fa2a84c8 +761, 0x27a1baece647f2cb +762, 0xfddf0cb92fe09dc3 +763, 0xeb5008fe965d8d96 +764, 0x4a3307937eb2e5c8 +765, 0xd07d74c240c6c363 +766, 0x16f62290179d1bbf +767, 0xe99c9bcc9cb1ece7 +768, 0xc64f9be03c8a93be +769, 0x32659effaf666c1f +770, 0x4bb228cfb30b6672 +771, 0x98764870842068a5 +772, 0x5b12ef2d2cd8bdcc +773, 0xbc79d1c1b41f28b8 +774, 0x97a517cf3279fc9a +775, 0x34ffd46c1d4d6025 +776, 0x9c302307ee25c8f0 +777, 0x399604eed1f18a8 +778, 0x1c9b813c2043142a +779, 0x2944ea5e55267fe9 +780, 0x5a8a9f5e728ea667 +781, 0x30c8440adb804a0 +782, 0xee0e6b627099a937 +783, 0x3d50757ada3c52da +784, 0x4548916b32c813ab +785, 0x602a186fe5bf109b +786, 0xf0d440a2227ba304 +787, 0x5a10d4e0ca9ea32b +788, 0x6e5eb90da13ba64c +789, 0x4c6af8fd04241ab2 +790, 0xf9eb31d26e093006 +791, 0x5d674878839fe3ea +792, 0x1562b55b2484e47c +793, 0xa87188c099c1cb61 +794, 0xb7736b8aa02a3392 +795, 0x5f4b301125abb20f +796, 0x361d566984637f44 +797, 0x68c4b3feac8bd0c3 +798, 0x7066c634dd2503c1 +799, 0xfecbf7c9441eb6ea +800, 0xdbc26ae0fc81436b +801, 0x9ef3e2b48252e7a4 +802, 0x31a49b4c339b37c7 +803, 0xb01b2a83cf346cf4 +804, 0xc24dc2347f82fbe3 +805, 0x134cad272dcd410f +806, 0x61260742823ba59c +807, 0x53ac4c193a97c730 +808, 0x9207c9833af34b52 +809, 0xa72e7ee77078d1f5 +810, 0x2e6f6e1b05936885 +811, 0x783b99ce5dbf9464 +812, 0xfdfeb6f0d027bb44 +813, 0x40eeb27096f92b0 +814, 0x5ef96ff5d4a4521f +815, 0x5595806ae873718a +816, 0x67d449eecf4ca1c3 +817, 0xde837ab611364f3f +818, 0x7034c24d2b139be9 +819, 0xe21166603e0a9c86 +820, 0x935694435c1f0d51 +821, 0x6cb3bec90c126088 +822, 0x4096ef662b7a9f89 +823, 0xd2d85b8d238d8c15 +824, 0xa4ea533ce3ec59b2 +825, 0x3654729d80a2db29 +826, 0x214c4cc3906d29d4 +827, 0x201c447e7588e373 +828, 0xe8b8f0ae25f683eb +829, 0x6744aaf5754e38af +830, 0xd1ffb10d6f27a061 +831, 0xe536733a7b3a6c30 +832, 0x39f0f66e47cbf2c9 +833, 0x856a9593526fde2 +834, 0x2e2a817a0098ea4b +835, 0xc5e1eeb551a0e3d3 +836, 0x3f21e2f5e2d50b2 +837, 0x906af56c66dd9f8c +838, 0x30f6dbd70329fac8 +839, 0xc443dfddf3c01a60 +840, 0x7ab85d9aa9675470 +841, 0x8c9080bd39717bfc +842, 0x4b1ccdb3c3597f6f +843, 0x74e2542d70ab5d67 +844, 0xbb3d236aad00f74 +845, 0xcf3cadf9a2804774 +846, 0xe851d9750e42bd07 +847, 0xc0ad82029b1c371f +848, 0x7ee119eb552d6c07 +849, 0xd8024049bd1d784a +850, 0xfa67a899760363 +851, 0xaa7c2f438b178197 +852, 0xc473674a47ffe064 +853, 0x539fbe3fc674c270 +854, 0xdb48484748a76f3b +855, 0xc73b2b092060d +856, 0xa1d2a15345016f5d +857, 0x4d0fe8599f9bba47 +858, 0xa0edc275e6f8f1d1 +859, 0x40590a8655bc8d72 +860, 0x35b4223161f05f75 +861, 0xa04c0c0f616752dc +862, 0x7f371ed2ca45432d +863, 0x2ff1a08f75ac6438 +864, 0xe2dc5c3682282f48 +865, 0xe1e4179fa98d9013 +866, 0x8cb083d6843a73d5 +867, 0xb4c2b5921b706854 +868, 0x738e14c0e7352445 +869, 0xcd2b646f91afd8c7 +870, 0xd5779a5b57a264fd +871, 0xc39ff855586c7d07 +872, 0x3e3f0098c631a859 +873, 0x644e02fae032110 +874, 0xa8834613c0a45278 +875, 0x69482f2c08e10657 +876, 0xe4ee475bdb87e69a +877, 0xdc1ef7b25c0d0019 +878, 0x88a3fa2be18d8744 +879, 0x60a02e0b21c5bec7 +880, 0xb6867b88aa19bc1a +881, 0xb599409affcf10eb +882, 0xaeaa1778a5e59daa +883, 0xd7a91a52c16663e3 +884, 0x93cb269affe07b1c +885, 0x841b6ced3a4ba815 +886, 0x84541768e1540a5c +887, 0xe3943c84f83b3020 +888, 0x5de366fbd7b45258 +889, 0xd787cc3bde91a661 +890, 0x814071446edecb57 +891, 0x15d8c602a1141514 +892, 0x72f07bc8002d1d0d +893, 0x4a8bd8dc9a1f0f3e +894, 0x8723796ae0f20d35 +895, 0xda7283c2051f73b2 +896, 0x2df0cc247f90bd3b +897, 0x79a8522b968f990a +898, 0x951ede190c8b9d02 +899, 0xc512f1a5b14b018a +900, 0xf0e3ddc03b9a4259 +901, 0x8cf4a35ad312e15f +902, 0xebef28926b11094b +903, 0x5628ba687325921c +904, 0xc3aa75e57edc49c3 +905, 0xc38382fa98e762ba +906, 0x8d209e896285848e +907, 0x2c7d6adf592b4a3e +908, 0x62de48e36f8338f3 +909, 0x4a752741e00de30e +910, 0xf7855b70f1f6ec2b +911, 0xa505fa4428199e43 +912, 0xe8b6b423b826bbac +913, 0x4bd1206cf8786d05 +914, 0x6dcf040391fe3bf4 +915, 0x913f500f87e1bba3 +916, 0x5acf775aa180a5d5 +917, 0x74dd28d9432ce739 +918, 0x996c2ff2f0dc2495 +919, 0x73dbfe6c56effe4 +920, 0x56fddd25196f5e40 +921, 0xe87810158f5b7 +922, 0x7b8795e996383f1f +923, 0x9ba5ee7c777c4c82 +924, 0x17ce3908d270fe1c +925, 0x3df9e613c1aedfae +926, 0xcdd26871b32fc8e1 +927, 0xd71cb13afc633979 +928, 0x63427c8ea9b1c79e +929, 0xd070f7664d3b405d +930, 0x46f2a9e32d9fb769 +931, 0xb4c3822a45e9fe9b +932, 0x8ba30b97fe6f5ec7 +933, 0x70aa554ee2fc11f9 +934, 0xa80c99dbe0cfcfaf +935, 0x36d9250cb2d68ed +936, 0x2995e4b9e1cd1db4 +937, 0x4b3803ba57fc570f +938, 0xae3959e7d740eaa5 +939, 0xb4cbd6662adbae08 +940, 0xae46576446e8dbc4 +941, 0xc4828e008a9a8a54 +942, 0x145d7db8e6554b2f +943, 0x1b1b8916a730c371 +944, 0xdaf84b2bebe31963 +945, 0x5b59b80ef23a2403 +946, 0x9180c7e89cab6fd3 +947, 0x80e58f5411babf34 +948, 0xa06cf55185b9b005 +949, 0x13b2c798424173ad +950, 0xc510f8e706311d49 +951, 0x1f974b83b6046d3a +952, 0xae6e8e85e822d1c3 +953, 0x66f2c8dc3274a31a +954, 0x7e04dbcbf65bd377 +955, 0xabf41ede01ec20a4 +956, 0x5efa0948f6bbb2ea +957, 0xbc91c99d8592255 +958, 0xf6d6917911d86d75 +959, 0x85ce273d54e9097a +960, 0xbdfd30f2420fff92 +961, 0x8802f02f610b537c +962, 0xd1d70037ed543229 +963, 0x908aaf97f9693a46 +964, 0x1f6cfeaa0834d53a +965, 0xa453fd1648ce04d2 +966, 0x2c38bb85ebc64af9 +967, 0xd2daff551c90c4f8 +968, 0xae5a0d949797d784 +969, 0xf0974c8552ac9593 +970, 0xa10b70499f65c693 +971, 0x39a449ebd594ddff +972, 0x8ea090f2b17b9b49 +973, 0xc592de318090fd83 +974, 0xb63e4fbc467b6912 +975, 0x57a0c1c5ce0e4dcc +976, 0xa7c517cf3d436b35 +977, 0xef6dcb0f3fad038b +978, 0xaf4fb60315b91287 +979, 0x5e0776f67304f331 +980, 0xe927753b8e6f7932 +981, 0xd3df2dd92559e304 +982, 0xdaed52aa6af44413 +983, 0x1b59f4dac1e181f8 +984, 0x4a73c2293877ef39 +985, 0xca45d0d015fe44de +986, 0x4659c8b7853735a8 +987, 0x12de6466bdf8adeb +988, 0xaeea857a09bfec15 +989, 0xcc9cf4b3c0b88a23 +990, 0xa44ae52396a5e1bf +991, 0x5847a724305d137f +992, 0x8f4d4de223956182 +993, 0x58254dfada867a8 +994, 0x900a98222c2f339e +995, 0xdb575260935d51d5 +996, 0x13fb4bfbbc0d7b53 +997, 0x62213850186bb92b +998, 0x2a34823312c00388 +999, 0x6148329042f743b0 diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/data/sfc64-testset-1.csv b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/data/sfc64-testset-1.csv new file mode 100644 index 0000000000000000000000000000000000000000..4fffe69591fea3f399f35034057096089d4017fb --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/data/sfc64-testset-1.csv @@ -0,0 +1,1001 @@ +seed, 0xdeadbeaf +0, 0xa475f55fbb6bc638 +1, 0xb2d594b6c29d971c +2, 0x275bc4ece4484fb1 +3, 0x569be72d9b3492fb +4, 0x89a5bb9b206a670c +5, 0xd951bfa06afdc3f9 +6, 0x7ee2e1029d52a265 +7, 0x12ef1d4de0cb4d4c +8, 0x41658ba8f0ef0280 +9, 0x5b650c82e4fe09c5 +10, 0x638a9f3e30ec4e94 +11, 0x147487fb2ba9233e +12, 0x89ef035603d2d1fb +13, 0xe66ca57a190e6cbe +14, 0x330f673740dd61fc +15, 0xc71d3dce2f8bb34e +16, 0x3c07c39ff150b185 +17, 0x5df952b6cae8f099 +18, 0x9f09f2b1f0ceac80 +19, 0x19598eee2d0c4c67 +20, 0x64e06483702e0ebd +21, 0xda04d1fdb545f7fa +22, 0xf2cf53b61a0c4f9b +23, 0xf0bb724ce196f66e +24, 0x71cefde55d9cf0f +25, 0x6323f62824a20048 +26, 0x1e93604680f14b4e +27, 0xd9d8fad1d4654025 +28, 0xf4ee25af2e76ca08 +29, 0x6af3325896befa98 +30, 0xad9e43abf5e04053 +31, 0xbf930e318ce09de3 +32, 0x61f9583b4f9ffe76 +33, 0x9b69d0b3d5ec8958 +34, 0xa608f250f9b2ca41 +35, 0x6fdba7073dc2bb5d +36, 0xa9d57601efea6d26 +37, 0xc24a88a994954105 +38, 0xc728b1f78d88fe5b +39, 0x88da88c2b083b3b2 +40, 0xa9e27f7303c76cfd +41, 0xc4c24608c29176eb +42, 0x5420b58466b972fd +43, 0xd2018a661b6756c8 +44, 0x7caed83d9573fc7 +45, 0x562a3d81b849a06a +46, 0x16588af120c21f2c +47, 0x658109a7e0eb4837 +48, 0x877aabb14d3822e1 +49, 0x95704c342c3745fe +50, 0xeeb8a0dc81603616 +51, 0x431bf94889290419 +52, 0xe4a9410ab92a5863 +53, 0xbc6be64ea60f12ba +54, 0x328a2da920015063 +55, 0x40f6b3bf8271ae07 +56, 0x4068ff00a0e854f8 +57, 0x1b287572ca13fa78 +58, 0xa11624a600490b99 +59, 0x4a04ef29eb7150fa +60, 0xcc9469ab5ffb739 +61, 0x99a6a9f8d95e782 +62, 0x8e90356573e7a070 +63, 0xa740b8fb415c81c4 +64, 0x47eccef67447f3da +65, 0x2c720afe3a62a49b +66, 0xe2a747f0a43eacf4 +67, 0xba063a87ab165576 +68, 0xbc1c78ed27feb5a3 +69, 0x285a19fa3974f9d +70, 0x489c61e704f5f0e3 +71, 0xf5ab04f6b03f238b +72, 0x7e25f88138a110dd +73, 0xc3d1cef3d7c1f1d1 +74, 0xc3de6ec64d0d8e00 +75, 0x73682a15b6cc5088 +76, 0x6fecbeb319163dc5 +77, 0x7e100d5defe570a1 +78, 0xad2af9af076dce57 +79, 0x3c65100e23cd3a9a +80, 0x4b442cc6cfe521bb +81, 0xe89dc50f8ab1ef75 +82, 0x8b3c6fdc2496566 +83, 0xdfc50042bc2c308c +84, 0xe39c5f158b33d2b2 +85, 0x92f6adefdfeb0ac +86, 0xdf5808a949c85b3e +87, 0x437384021c9dace9 +88, 0xa7b5ed0d3d67d8f +89, 0xe1408f8b21da3c34 +90, 0xa1bba125c1e80522 +91, 0x7611dc4710385264 +92, 0xb00a46ea84082917 +93, 0x51bf8002ffa87cef +94, 0x9bb81013e9810adc +95, 0xd28f6600013541cd +96, 0xc2ca3b1fa7791c1f +97, 0x47f9ad58f099c82c +98, 0x4d1bb9458469caf9 +99, 0xca0b165b2844257 +100, 0xc3b2e667d075dc66 +101, 0xde22f71136a3dbb1 +102, 0x23b4e3b6f219e4c3 +103, 0x327e0db4c9782f66 +104, 0x9365506a6c7a1807 +105, 0x3e868382dedd3be7 +106, 0xff04fa6534bcaa99 +107, 0x96621a8862995305 +108, 0x81bf39cb5f8e1df7 +109, 0x79b684bb8c37af7a +110, 0xae3bc073c3cde33c +111, 0x7805674112c899ac +112, 0xd95a27995abb20f2 +113, 0x71a503c57b105c40 +114, 0x5ff00d6a73ec8acc +115, 0x12f96391d91e47c2 +116, 0xd55ca097b3bd4947 +117, 0x794d79d20468b04 +118, 0x35d814efb0d7a07d +119, 0xfa9ac9bd0aae76d3 +120, 0xa77b8a3711e175cd +121, 0xe6694fbf421f9489 +122, 0xd8f1756525a1a0aa +123, 0xe38dfa8426277433 +124, 0x16b640c269bbcd44 +125, 0x2a7a5a67ca24cfeb +126, 0x669039c28d5344b4 +127, 0x2a445ee81fd596bb +128, 0x600df94cf25607e0 +129, 0x9358561a7579abff +130, 0xee1d52ea179fc274 +131, 0x21a8b325e89d31be +132, 0x36fc0917486eec0a +133, 0x3d99f40717a6be9f +134, 0x39ac140051ca55ff +135, 0xcef7447c26711575 +136, 0xf22666870eff441d +137, 0x4a53c6134e1c7268 +138, 0xd26de518ad6bdb1b +139, 0x1a736bf75b8b0e55 +140, 0xef1523f4e6bd0219 +141, 0xb287b32fd615ad92 +142, 0x2583d6af5e841dd5 +143, 0x4b9294aae7ca670c +144, 0xf5aa4a84174f3ca9 +145, 0x886300f9e0dc6376 +146, 0x3611401e475ef130 +147, 0x69b56432b367e1ac +148, 0x30c330e9ab36b7c4 +149, 0x1e0e73079a85b8d5 +150, 0x40fdfc7a5bfaecf +151, 0xd7760f3e8e75a085 +152, 0x1cc1891f7f625313 +153, 0xeece1fe6165b4272 +154, 0xe61111b0c166a3c1 +155, 0x2f1201563312f185 +156, 0xfd10e8ecdd2a57cb +157, 0x51cdc8c9dd3a89bf +158, 0xed13cc93938b5496 +159, 0x843816129750526b +160, 0xd09995cd6819ada +161, 0x4601e778d40607df +162, 0xef9df06bd66c2ea0 +163, 0xae0bdecd3db65d69 +164, 0xbb921a3c65a4ae9a +165, 0xd66698ce8e9361be +166, 0xacdc91647b6068f4 +167, 0xe505ef68f2a5c1c0 +168, 0xd6e62fd27c6ab137 +169, 0x6a2ba2c6a4641d86 +170, 0x9c89143715c3b81 +171, 0xe408c4e00362601a +172, 0x986155cbf5d4bd9d +173, 0xb9e6831728c893a7 +174, 0xb985497c3bf88d8c +175, 0xd0d729214b727bec +176, 0x4e557f75fece38a +177, 0x6572067fdfd623ca +178, 0x178d49bb4d5cd794 +179, 0xe6baf59f60445d82 +180, 0x5607d53518e3a8d2 +181, 0xba7931adb6ebbd61 +182, 0xe853576172611329 +183, 0xe945daff96000c44 +184, 0x565b9ba3d952a176 +185, 0xcdb54d4f88c584c8 +186, 0x482a7499bee9b5e5 +187, 0x76560dd0affe825b +188, 0x2a56221faa5ca22c +189, 0x7729be5b361f5a25 +190, 0xd6f2195795764876 +191, 0x59ef7f8f423f18c5 +192, 0x7ebefed6d02adde1 +193, 0xcfec7265329c73e5 +194, 0x4fd8606a5e59881c +195, 0x95860982ae370b73 +196, 0xdecfa33b1f902acc +197, 0xf9b8a57400b7c0a6 +198, 0xd20b822672ec857b +199, 0x4eb81084096c7364 +200, 0xe535c29a44d9b6ad +201, 0xdef8b48ebacb2e29 +202, 0x1063bc2b8ba0e915 +203, 0xe4e837fb53d76d02 +204, 0x4df935db53579fb8 +205, 0xa30a0c8053869a89 +206, 0xe891ee58a388a7b5 +207, 0x17931a0c64b8a985 +208, 0xaf2d350b494ce1b3 +209, 0x2ab9345ffbcfed82 +210, 0x7de3fe628a2592f0 +211, 0x85cf54fab8b7e79d +212, 0x42d221520edab71b +213, 0x17b695b3af36c233 +214, 0xa4ffe50fe53eb485 +215, 0x1102d242db800e4d +216, 0xc8dc01f0233b3b6 +217, 0x984a030321053d36 +218, 0x27fa8dc7b7112c0e +219, 0xba634dd8294e177f +220, 0xe67ce34b36332eb +221, 0x8f1351e1894fb41a +222, 0xb522a3048761fd30 +223, 0xc350ad9bc6729edc +224, 0xe0ed105bd3c805e1 +225, 0xa14043d2b0825aa7 +226, 0xee7779ce7fc11fdf +227, 0xc0fa8ba23a60ab25 +228, 0xb596d1ce259afbad +229, 0xaa9b8445537fdf62 +230, 0x770ab2c700762e13 +231, 0xe812f1183e40cc1 +232, 0x44bc898e57aefbbd +233, 0xdd8a871df785c996 +234, 0x88836a5e371eb36b +235, 0xb6081c9152623f27 +236, 0x895acbcd6528ca96 +237, 0xfb67e33ddfbed435 +238, 0xaf7af47d323ce26 +239, 0xe354a510c3c39b2d +240, 0x5cacdedda0672ba3 +241, 0xa440d9a2c6c22b09 +242, 0x6395099f48d64304 +243, 0xc11cf04c75f655b5 +244, 0x1c4e054d144ddb30 +245, 0x3e0c2db89d336636 +246, 0x127ecf18a5b0b9a7 +247, 0x3b50551a88ea7a73 +248, 0xbd27003e47f1f684 +249, 0xf32d657782baac9b +250, 0x727f5cabf020bc9 +251, 0x39c1c1c226197dc7 +252, 0x5552c87b35deeb69 +253, 0x64d54067b5ce493f +254, 0x3494b091fe28dda0 +255, 0xdf0278bc85ee2965 +256, 0xdef16fec25efbd66 +257, 0xe2be09f578c4ce28 +258, 0xd27a9271979d3019 +259, 0x427f6fcd71845e3 +260, 0x26b52c5f81ec142b +261, 0x98267efc3986ad46 +262, 0x7bf4165ddb7e4374 +263, 0xd05f7996d7941010 +264, 0x3b3991de97b45f14 +265, 0x9068217fb4f27a30 +266, 0xd8fe295160afc7f3 +267, 0x8a159fab4c3bc06f +268, 0x57855506d19080b6 +269, 0x7636df6b3f2367a4 +270, 0x2844ee3abd1d5ec9 +271, 0xe5788de061f51c16 +272, 0x69e78cc9132a164 +273, 0xacd53cde6d8cd421 +274, 0xb23f3100068e91da +275, 0x4140070a47f53891 +276, 0xe4a422225a96e53a +277, 0xb82a8925a272a2ac +278, 0x7c2f9573590fe3b7 +279, 0xbaf80764db170575 +280, 0x955abffa54358368 +281, 0x355ce7460614a869 +282, 0x3700ede779a4afbf +283, 0x10a6ec01d92d68cd +284, 0x3308f5a0a4c0afef +285, 0x97b892d7601136c9 +286, 0x4955c3b941b8552e +287, 0xca85aa67e941961d +288, 0xb1859ae5db28e9d2 +289, 0x305d072ac1521fbd +290, 0xed52a868996085bb +291, 0x723bfa6a76358852 +292, 0x78d946ecd97c5fb3 +293, 0x39205b30a8e23e79 +294, 0xb927e3d086baadbe +295, 0xa18d6946136e1ff5 +296, 0xdab6f0b51c1eb5ff +297, 0xf0a640bf7a1af60c +298, 0xf0e81db09004d0d4 +299, 0xfe76cebdbe5a4dde +300, 0x2dafe9cc3decc376 +301, 0x4c871fdf1af34205 +302, 0xe79617d0c8fa893b +303, 0xee658aaad3a141f7 +304, 0xfd91aa74863e19f1 +305, 0x841b8f55c103cc22 +306, 0x22766ed65444ad5d +307, 0x56d03d1beca6c17a +308, 0x5fd4c112c92036ae +309, 0x75466ae58a5616dc +310, 0xfbf98b1081e802a9 +311, 0xdc325e957bf6d8f5 +312, 0xb08da7015ebd19b7 +313, 0xf25a9c0944f0c073 +314, 0xf4625bafa0ced718 +315, 0x4349c9e093a9e692 +316, 0x75a9ccd4dd8935cb +317, 0x7e6cf9e539361e91 +318, 0x20fdd22fb6edd475 +319, 0x5973021b57c2311f +320, 0x75392403667edc15 +321, 0xed9b2156ea70d9f1 +322, 0xf40c114db50b64a0 +323, 0xe26bb2c9eef20c62 +324, 0x409c1e3037869f03 +325, 0xcdfd71fdda3b7f91 +326, 0xa0dfae46816777d6 +327, 0xde060a8f61a8deb8 +328, 0x890e082a8b0ca4fc +329, 0xb9f2958eddf2d0db +330, 0xd17c148020d20e30 +331, 0xffdc9cc176fe7201 +332, 0xffb83d925b764c1 +333, 0x817ea639e313da8d +334, 0xa4dd335dd891ca91 +335, 0x1342d25a5e81f488 +336, 0xfa7eb9c3cf466b03 +337, 0xfe0a423d44b185d0 +338, 0x101cfd430ab96049 +339, 0x7b5d3eda9c4504b +340, 0xe20ccc006e0193f1 +341, 0xf54ccddedebc5df0 +342, 0xc0edd142bd58f1db +343, 0x3831f40d378d2430 +344, 0x80132353f0a88289 +345, 0x688f23c419d03ef8 +346, 0x4c6837e697884066 +347, 0x699387bb2e9a3a8f +348, 0x8996f860342448d8 +349, 0xb0f80dff99bfa5cc +350, 0x3e927a7f9ea12c8e +351, 0xd7e498d1e5f9dff3 +352, 0x78ecb97bb3f864cc +353, 0x3c4ffd069a014d38 +354, 0xf8d5073a1e09b4d4 +355, 0x8717e854f9faef23 +356, 0xfbcc5478d8d0ad7 +357, 0xd3cd8b233ca274ff +358, 0x8bd8f11f79beb265 +359, 0xf64498a832d8fd0e +360, 0xb01bba75112131ec +361, 0x55572445a7869781 +362, 0x7b56622f18cb3d7a +363, 0x7f192c9e075bdb83 +364, 0xd9a112f836b83ff3 +365, 0x68673b37269653dc +366, 0xe46a9433fb6a0879 +367, 0x127d756ca4779001 +368, 0xc1378e8b1e8eab94 +369, 0x1006edb0f51d078c +370, 0xc6dd53961232d926 +371, 0x9a4aeef44038256d +372, 0xd357f4fa652d4f5f +373, 0x59f3d2cc3378598 +374, 0xe76e6207a824a7fc +375, 0x5fc5e33712ceffef +376, 0x77d24aeb0ccb1adc +377, 0x5be4b2826805659e +378, 0x257c69d787e64634 +379, 0x58dd52ca6bc727b1 +380, 0x3ab997767235ea33 +381, 0x986a2a7a966fad14 +382, 0xc900f8b27761dcc4 +383, 0x44991bdb13795700 +384, 0xe5c145a4fe733b2 +385, 0x56f041b56bffe0d3 +386, 0x5779c4fef8067996 +387, 0xa0fe8748e829532d +388, 0x840c1277d78d9dd4 +389, 0x37ebcb315432acbc +390, 0xf4bc8738433ba3be +391, 0x8b122993f2e10062 +392, 0xe1fe8481f2681ed5 +393, 0x8e23f1630d9f494a +394, 0xda24661a01b7d0b3 +395, 0x7a02942a179cee36 +396, 0xf1e08a3c09b71ac +397, 0x3dec2cc7ee0bd8fd +398, 0x1f3e480113d805d4 +399, 0xc061b973ad4e3f2c +400, 0x6bea750f17a66836 +401, 0xbc2add72eac84c25 +402, 0xcff058d3f97934ca +403, 0x54ccc30987778ec2 +404, 0x93449ec1e1469558 +405, 0xe2ff369eb0c6836 +406, 0x41c2df2d63bf8e55 +407, 0xf9302629b6c71be2 +408, 0xdd30376b8e5ab29a +409, 0x12db9e04f911d754 +410, 0x8d03d6cd359f1b97 +411, 0xe15956511abf1cee +412, 0x9b68e10e2c2fd940 +413, 0x2e28de6491c1ce53 +414, 0x52b329b72d0c109d +415, 0xc2c0b115f9da2a60 +416, 0x6ca084105271bbff +417, 0x49b92b8676058c1e +418, 0x767fc92a70f7e5a3 +419, 0x87ba4ed4b65a6aa0 +420, 0xf70b052e0a3975e9 +421, 0x3e925c3306db9eec +422, 0x43253f1d96ac9513 +423, 0xe3e04f1a1ea454c4 +424, 0x763e3f4cc81ba0c8 +425, 0x2a2721ac69265705 +426, 0xdf3b0ac6416ea214 +427, 0xa6a6b57450f3e000 +428, 0xc3d3b1ac7dbfe6ac +429, 0xb66e5e6f7d2e4ec0 +430, 0x43c65296f98f0f04 +431, 0xdb0f6e3ff974d842 +432, 0x3d6b48e02ebb203b +433, 0xd74674ebf09d8f27 +434, 0xbe65243c58fc1200 +435, 0x55eb210a68d42625 +436, 0x87badab097dbe883 +437, 0xada3fda85a53824f +438, 0xef2791e8f48cd37a +439, 0x3fe7fceb927a641a +440, 0xd3bffd3ff031ac78 +441, 0xb94efe03da4d18fb +442, 0x162a0ad8da65ea68 +443, 0x300f234ef5b7e4a6 +444, 0xa2a8b4c77024e4fb +445, 0x5950f095ddd7b109 +446, 0xded66dd2b1bb02ba +447, 0x8ec24b7fa509bcb6 +448, 0x9bede53d924bdad6 +449, 0xa9c3f46423be1930 +450, 0x6dfc90597f8de8b4 +451, 0xb7419ebc65b434f0 +452, 0xa6596949238f58b9 +453, 0x966cbade640829b8 +454, 0x58c74877bdcbf65e +455, 0xaa103b8f89b0c453 +456, 0x219f0a86e41179a4 +457, 0x90f534fc06ddc57f +458, 0x8db7cdd644f1affa +459, 0x38f91de0167127ac +460, 0xdcd2a65e4df43daa +461, 0x3e04f34a7e01f834 +462, 0x5b237eea68007768 +463, 0x7ff4d2b015921768 +464, 0xf786b286549d3d51 +465, 0xaefa053fc2c3884c +466, 0x8e6a8ff381515d36 +467, 0x35b94f3d0a1fce3c +468, 0x165266d19e9abb64 +469, 0x1deb5caa5f9d8076 +470, 0x13ab91290c7cfe9d +471, 0x3651ca9856be3e05 +472, 0xe7b705f6e9cccc19 +473, 0xd6e7f79668c127ed +474, 0xa9faf37154896f92 +475, 0x89fbf190603e0ab1 +476, 0xb34d155a86f942d0 +477, 0xb2d4400a78bfdd76 +478, 0x7c0946aca8cfb3f0 +479, 0x7492771591c9d0e8 +480, 0xd084d95c5ca2eb28 +481, 0xb18d12bd3a6023e +482, 0xea217ed7b864d80b +483, 0xe52f69a755dd5c6f +484, 0x127133993d81c4aa +485, 0xe07188fcf1670bfb +486, 0x178fbfe668e4661d +487, 0x1c9ee14bb0cda154 +488, 0x8d043b96b6668f98 +489, 0xbc858986ec96ca2b +490, 0x7660f779d528b6b7 +491, 0xd448c6a1f74ae1d3 +492, 0x178e122cfc2a6862 +493, 0x236f000abaf2d23b +494, 0x171b27f3f0921915 +495, 0x4c3ff07652f50a70 +496, 0x18663e5e7d3a66ca +497, 0xb38c97946c750cc9 +498, 0xc5031aae6f78f909 +499, 0x4d1514e2925e95c1 +500, 0x4c2184a741dabfbb +501, 0xfd410364edf77182 +502, 0xc228157f863ee873 +503, 0x9856fdc735cc09fc +504, 0x660496cd1e41d60e +505, 0x2edf1d7e01954c32 +506, 0xd32e94639bdd98cf +507, 0x8e153f48709a77d +508, 0x89357f332d2d6561 +509, 0x1840d512c97085e6 +510, 0x2f18d035c9e26a85 +511, 0x77b88b1448b26d5b +512, 0xc1ca6ef4cdae0799 +513, 0xcc203f9e4508165f +514, 0xeaf762fbc9e0cbbe +515, 0xc070c687f3c4a290 +516, 0xd49ed321068d5c15 +517, 0x84a55eec17ee64ee +518, 0x4d8ee685298a8871 +519, 0x9ff5f17d7e029793 +520, 0x791d7d0d62e46302 +521, 0xab218b9114e22bc6 +522, 0x4902b7ab3f7119a7 +523, 0x694930f2e29b049e +524, 0x1a3c90650848999f +525, 0x79f1b9d8499c932b +526, 0xfacb6d3d55e3c92f +527, 0x8fd8b4f25a5da9f5 +528, 0xd037dcc3a7e62ae7 +529, 0xfecf57300d8f84f4 +530, 0x32079b1e1dc12d48 +531, 0xe5f8f1e62b288f54 +532, 0x97feba3a9c108894 +533, 0xd279a51e1899a9a0 +534, 0xd68eea8e8e363fa8 +535, 0x7394cf2deeca9386 +536, 0x5f70b0c80f1dbf10 +537, 0x8d646916ed40462 +538, 0xd253bb1c8a12bbb6 +539, 0x38f399a821fbd73e +540, 0x947523a26333ac90 +541, 0xb52e90affbc52a37 +542, 0xcf899cd964654da4 +543, 0xdf66ae9cca8d99e7 +544, 0x6051478e57c21b6a +545, 0xffa7dc975af3c1da +546, 0x195c7bff2d1a8f5 +547, 0x64f12b6575cf984d +548, 0x536034cb842cf9e1 +549, 0x180f247ce5bbfad +550, 0x8ced45081b134867 +551, 0x532bbfdf426710f3 +552, 0x4747933e74c4f54d +553, 0x197a890dc4793401 +554, 0x76c7cc2bd42fae2 +555, 0xdabfd67f69675dd0 +556, 0x85c690a68cdb3197 +557, 0xe482cec89ce8f92 +558, 0x20bc9fb7797011b1 +559, 0x76dc85a2185782ad +560, 0x3df37c164422117a +561, 0x99211f5d231e0ab0 +562, 0xef7fd794a0a91f4 +563, 0x419577151915f5fe +564, 0x3ce14a0a7135dae3 +565, 0x389b57598a075d6a +566, 0x8cc2a9d51b5af9aa +567, 0xe80a9beffbd13f13 +568, 0x65e96b22ea8a54d8 +569, 0x79f38c4164138ede +570, 0xd1955846cba03d81 +571, 0x60359fe58e4f26d6 +572, 0x4ea724f585f8d13e +573, 0x316dfdbadc801a3c +574, 0x20aa29b7c6dd66fe +575, 0x65eaf83a6a008caa +576, 0x407000aff1b9e8cb +577, 0xb4d49bfb2b268c40 +578, 0xd4e6fe8a7a0f14a9 +579, 0xe34afef924e8f58e +580, 0xe377b0c891844824 +581, 0x29c2e20c112d30c8 +582, 0x906aad1fe0c18a95 +583, 0x308385f0efbb6474 +584, 0xf23900481bf70445 +585, 0xfdfe3ade7f937a55 +586, 0xf37aae71c33c4f97 +587, 0x1c81e3775a8bed85 +588, 0x7eb5013882ce35ea +589, 0x37a1c1692495818d +590, 0x3f90ae118622a0ba +591, 0x58e4fe6fea29b037 +592, 0xd10ff1d269808825 +593, 0xbce30edb60c21bba +594, 0x123732329afd6fee +595, 0x429b4059f797d840 +596, 0x421166568a8c4be1 +597, 0x88f895c424c1bd7f +598, 0x2adaf7a7b9f781cb +599, 0xa425644b26cb698 +600, 0x8cc44d2486cc5743 +601, 0xdb9f357a33abf6ba +602, 0x1a57c4ea77a4d70c +603, 0x1dea29be75239e44 +604, 0x463141a137121a06 +605, 0x8fecfbbe0b8a9517 +606, 0x92c83984b3566123 +607, 0x3b1c69180ed28665 +608, 0x14a6073425ea8717 +609, 0x71f4c2b3283238d7 +610, 0xb3d491e3152f19f +611, 0x3a0ba3a11ebac5d2 +612, 0xddb4d1dd4c0f54ac +613, 0xdb8f36fe02414035 +614, 0x1cf5df5031b1902c +615, 0x23a20ed12ef95870 +616, 0xf113e573b2dedcbb +617, 0x308e2395cde0a9fa +618, 0xd377a22581c3a7da +619, 0xe0ced97a947a66fb +620, 0xe44f4de9cd754b00 +621, 0x2344943337d9d1bf +622, 0x4b5ae5e2ea6e749c +623, 0x9b8d2e3ef41d1c01 +624, 0x59a5a53ebbd24c6b +625, 0x4f7611bf9e8a06fb +626, 0xea38c7b61361cd06 +627, 0xf125a2bfdd2c0c7 +628, 0x2df8dcb5926b9ebb +629, 0x233e18720cc56988 +630, 0x974c61379b4aa95e +631, 0xc7fe24c1c868910b +632, 0x818fd1affc82a842 +633, 0xcee92a952a26d38e +634, 0x8962f575ebcbf43 +635, 0x7770687e3678c460 +636, 0xdfb1db4ed1298117 +637, 0xb9db54cb03d434d3 +638, 0x34aebbf2244257ad +639, 0xd836db0cb210c490 +640, 0x935daed7138957cd +641, 0x3cd914b14e7948fd +642, 0xd0472e9ed0a0f7f0 +643, 0xa9df33dca697f75e +644, 0x15e9ea259398721a +645, 0x23eeba0f970abd60 +646, 0x2217fdf8bbe99a12 +647, 0x5ea490a95717b198 +648, 0xf4e2bfc28280b639 +649, 0x9d19916072d6f05c +650, 0x5e0387cab1734c6a +651, 0x93c2c8ac26e5f01e +652, 0xb0d934354d957eb1 +653, 0xee5099a1eef3188c +654, 0x8be0abca8edc1115 +655, 0x989a60845dbf5aa3 +656, 0x181c7ed964eee892 +657, 0x49838ea07481288d +658, 0x17dbc75d66116b2e +659, 0xa4cafb7a87c0117e +660, 0xab2d0ae44cdc2e6e +661, 0xdf802f2457e7da6 +662, 0x4b966c4b9187e124 +663, 0x62de9db6f4811e1a +664, 0x1e20485968bc62 +665, 0xe9ac288265caca94 +666, 0xc5c694d349aa8c1a +667, 0x3d67f2083d9bdf10 +668, 0x9a2468e503085486 +669, 0x9d6acd3dc152d1a3 +670, 0xca951e2aeee8df77 +671, 0x2707371af9cdd7b0 +672, 0x2347ae6a4eb5ecbd +673, 0x16abe5582cb426f +674, 0x523af4ff980bbccb +675, 0xb07a0f043e3694aa +676, 0x14d7c3da81b2de7 +677, 0xf471f1b8ac22305b +678, 0xdb087ffff9e18520 +679, 0x1a352db3574359e8 +680, 0x48d5431502cc7476 +681, 0x7c9b7e7003dfd1bf +682, 0x4f43a48aae987169 +683, 0x9a5d3eb66dedb3e9 +684, 0xa7b331af76a9f817 +685, 0xba440154b118ab2d +686, 0x64d22344ce24c9c6 +687, 0xa22377bd52bd043 +688, 0x9dfa1bb18ca6c5f7 +689, 0xdccf44a92f644c8b +690, 0xf623d0a49fd18145 +691, 0x556d5c37978e28b3 +692, 0xad96e32ce9d2bb8b +693, 0x2e479c120be52798 +694, 0x7501cf871af7b2f7 +695, 0xd02536a5d026a5b8 +696, 0x4b37ff53e76ab5a4 +697, 0xdb3a4039caaeab13 +698, 0x6cbd65e3b700c7be +699, 0x7367abd98761a147 +700, 0xf4f9ba216a35aa77 +701, 0xf88ca25ce921eb86 +702, 0xb211de082ec2cbf2 +703, 0xdd94aa46ec57e12e +704, 0xa967d74ad8210240 +705, 0xdaa1fada8cfa887 +706, 0x85901d081c4488ee +707, 0xcf67f79a699ef06 +708, 0x7f2f1f0de921ee14 +709, 0x28bc61e9d3f2328b +710, 0x3332f2963faf18e5 +711, 0x4167ac71fcf43a6 +712, 0x843c1746b0160b74 +713, 0xd9be80070c578a5e +714, 0xbd7250c9af1473e7 +715, 0x43f78afaa3647899 +716, 0x91c6b5dd715a75a5 +717, 0x29cc66c8a07bfef3 +718, 0x3f5c667311dc22be +719, 0x4f49cd47958260cd +720, 0xbef8be43d920b64e +721, 0x7a892a5f13061d8b +722, 0x9532f40125c819b1 +723, 0x924fca3045f8a564 +724, 0x9b2c6442453b0c20 +725, 0x7e21009085b8e793 +726, 0x9b98c17e17af59d2 +727, 0xba61acb73e3ae89a +728, 0xb9d61a710555c138 +729, 0xc2a425d80978974b +730, 0xa275e13592da7d67 +731, 0xe962103202d9ad0f +732, 0xbdf8367a4d6f33fd +733, 0xe59beb2f8648bdc8 +734, 0xb4c387d8fbc4ac1c +735, 0x5e3f276b63054b75 +736, 0xf27e616aa54d8464 +737, 0x3f271661d1cd7426 +738, 0x43a69dbee7502c78 +739, 0x8066fcea6df059a1 +740, 0x3c10f19409bdc993 +741, 0x6ba6f43fb21f23e0 +742, 0x9e182d70a5bccf09 +743, 0x1520783d2a63a199 +744, 0xba1dcc0c70b9cace +745, 0x1009e1e9b1032d8 +746, 0xf632f6a95fb0315 +747, 0x48e711c7114cbfff +748, 0xef281dcec67debf7 +749, 0x33789894d6abf59b +750, 0x6c8e541fffbe7f9c +751, 0x85417f13b08e0a88 +752, 0x9a581e36d589608f +753, 0x461dca50b1befd35 +754, 0x5a3231680dde6462 +755, 0xcc57acf729780b97 +756, 0x50301efef62e1054 +757, 0x675d042cd4f6bbc9 +758, 0x1652fdd3794384c9 +759, 0x1c93bbeeb763cd4d +760, 0x44b7240c4b105242 +761, 0x4c6af2a1b606ccfb +762, 0x18fc43ece2ec1a40 +763, 0x859a5511aeae8acb +764, 0x2f56826f1996ad2f +765, 0xa8e95ce8bb363bdf +766, 0xf4da396054e50e4b +767, 0x5493865e9895883c +768, 0x768e4c8b332ac0e3 +769, 0x32195d2aa583fca5 +770, 0xf2f353f21266bc15 +771, 0x43cddf1d021307d +772, 0x6031e3aa30300e4a +773, 0x4f1298469ac6088f +774, 0x4b4d450bafac574e +775, 0x23e1cf9c0582a22b +776, 0x2e9036980db49cd0 +777, 0xe4e228b113c411b2 +778, 0x8bddcdb82b51706 +779, 0xd2a7ea8288593629 +780, 0x67fe90e98fdda61 +781, 0x7b63494dba95717b +782, 0x105625904510d782 +783, 0xdf4aa2242454e50a +784, 0x32541d6cd7d6c7e3 +785, 0x5661fb432591cf3b +786, 0xce920a5ed047bce7 +787, 0xed4178a3c96eea8f +788, 0xe378cd996e39863b +789, 0x169e1fdc8e2b05e1 +790, 0xaee1812ef7149a96 +791, 0x648571c7453d12c5 +792, 0xb7b6bc9328573c43 +793, 0xe7fb969078e270d7 +794, 0xdfc2b1b8985f6e6f +795, 0x862b6527ee39a1aa +796, 0x1ee329aea91d7882 +797, 0x20d25324f2fe704 +798, 0xbfcc47401fc3bbfd +799, 0x1515cdc8d48b2904 +800, 0xbd6eefe86284261c +801, 0x9b1f28e3b35f22ee +802, 0x842a29d35e5aecda +803, 0xf2346109ad370765 +804, 0x24d68add5a71afd9 +805, 0x4a691421613d91e2 +806, 0x60e3058b3c244051 +807, 0x79194905cdaa5de8 +808, 0xe0e2df35c01e8987 +809, 0xe29b78beffbb5e4a +810, 0xcdcdbc020218c19e +811, 0x5ae0af8c16feae43 +812, 0x8109292feeaf14fa +813, 0x34113f7508dfa521 +814, 0xc062ac163f56730a +815, 0xf1660e66ec6d4c4c +816, 0x5966c55f60151c80 +817, 0x3865ae8ec934b17 +818, 0x472a7314afb055ec +819, 0x7a24277309a44a44 +820, 0x556e02dd35d38baa +821, 0x9849611a1bc96ec1 +822, 0xd176f5d5a8eb0843 +823, 0x44db12ec60510030 +824, 0x272e3a06a0030078 +825, 0x7c4764dbefc075ea +826, 0x910712f3735c1183 +827, 0xd49a2da74ae7aff6 +828, 0xcf9b3e6e8f776d71 +829, 0x27789fe3ec481a02 +830, 0x86659f82c6b5912b +831, 0xe044b3dbf339158c +832, 0x99d81f6bb62a37b0 +833, 0x5f5830c246fada9a +834, 0xe68abab1eeb432cb +835, 0x49c5c5ace04e104 +836, 0x1ac3871b3fc6771b +837, 0x773b39f32d070652 +838, 0x9c4138c2ae58b1f3 +839, 0xac41c63d7452ac60 +840, 0x9248826b245359e1 +841, 0x99bba1c7a64f1670 +842, 0xe0dc99ff4ebb92f2 +843, 0x113638652740f87c +844, 0xebf51e94da88cfc +845, 0x5441c344b81b2585 +846, 0xe1e69e0bc2de652a +847, 0xe9ab6d64ae42ed1e +848, 0x879af8730e305f31 +849, 0x36b9ad912c7e00d6 +850, 0x83ef5e9fca853886 +851, 0xda54d48bb20ea974 +852, 0x32c6d93aefa92aa2 +853, 0x4e887b2c3391847d +854, 0x50966e815f42b1b8 +855, 0x53411ac087832837 +856, 0x46f64fef79df4f29 +857, 0xb34aae3924cd272c +858, 0xf5ad455869a0adbe +859, 0x8351ded7144edac8 +860, 0xeb558af089677494 +861, 0x36ed71d69293a8d6 +862, 0x659f90bf5431b254 +863, 0x53349102b7519949 +864, 0x3db83e20b1713610 +865, 0x6d63f96090556254 +866, 0x4cc0467e8f45c645 +867, 0xb8840c4bd5cd4091 +868, 0xbd381463cc93d584 +869, 0x203410d878c2066d +870, 0x2ebea06213cf71c8 +871, 0x598e8fb75e3fceb4 +872, 0xdcca41ceba0fce02 +873, 0x61bf69212b56aae5 +874, 0x97eed7f70c9114fa +875, 0xf46f37a8b7a063f9 +876, 0x66c8f4ffe5bd6efa +877, 0xe43fd6efda2d4e32 +878, 0x12d6c799e5ad01de +879, 0x9ac83e7f8b709360 +880, 0xbbb7bb3c1957513d +881, 0x7f87c08d4b3796b0 +882, 0x9a7d1d74b6aa4a5c +883, 0xa4314530ff741b6f +884, 0x99a80c6b6f15fca8 +885, 0xd2fec81d6d5fc3ce +886, 0x15a98be1cc40cea +887, 0x98693eb7719366f3 +888, 0x36ccdc2a9e9d4de8 +889, 0x3c8208f63d77df25 +890, 0xca2e376e2343df6 +891, 0xcc9b17cbb54420c6 +892, 0x8724c44a64d7dcb8 +893, 0x9d00c6949ff33869 +894, 0xf4f8e584d2699372 +895, 0x88f4748cdd5a2d53 +896, 0xe215072a1205bc6d +897, 0x190934fe6d740442 +898, 0x7fac5c0ab2af106d +899, 0x1b86633a0bd84fa1 +900, 0x1293e54318492dfb +901, 0x433324fd390f34b9 +902, 0x4c5eb2c67a44643b +903, 0x59a6e281c388b0dd +904, 0xe78e03f9c44623b7 +905, 0x91307a93c768fc3d +906, 0xde8867b004d8e3ff +907, 0xdf52c3f57b7c5862 +908, 0x993f3e1d10358a92 +909, 0x9ccb10bc3e18662d +910, 0x45093ce48a114c73 +911, 0xd59d05979d26330a +912, 0x417c0e03300119a9 +913, 0x1c336500f90cde81 +914, 0x1c8ccd29ead9b85b +915, 0xb76baf3e55d4d950 +916, 0x133ad6196c75fd7e +917, 0x34200b0cde7ed560 +918, 0x9c7c3dacb213c8d9 +919, 0xd97563c4fd9bf1b6 +920, 0x5d910e871835b6cb +921, 0x7d46c4733a16bdf9 +922, 0xe41d73194ddc87b2 +923, 0x7d3d8a0855a465a9 +924, 0x70c2a8b5d3f90c0f +925, 0x9e7565ca5dccfe12 +926, 0x2c0acb4577aa51b1 +927, 0x3d2cd211145b79c7 +928, 0x15a7b17aa6da7732 +929, 0xab44a3730c27d780 +930, 0xf008bd6c802bde3a +931, 0x82ed86ddf3619f77 +932, 0xaabe982ab15c49f9 +933, 0x9bcad8fa6d8e58a4 +934, 0x8f39ed8243718aa1 +935, 0xe9489340e03e3cb6 +936, 0xc722314f5eefb8d0 +937, 0x870e8869a436df59 +938, 0x4dae75b8087a8204 +939, 0xe1d790f6ec6e425b +940, 0xafd39ea1b1d0ed09 +941, 0xdf2c99e464ddf08f +942, 0x74936d859ab9644d +943, 0x3871302164250e73 +944, 0x764b68921e911886 +945, 0x2a1d024b26bb9d66 +946, 0x797fba43918e75b4 +947, 0x62ec6d24ccca335b +948, 0xf4bd8b951762b520 +949, 0x9d450dede9119397 +950, 0x5393a26d10f8c124 +951, 0x6b74769392896b57 +952, 0x7f61dbcc0e328581 +953, 0x64e1df3884d0d94 +954, 0xba77dcdf23738c37 +955, 0xf8e288bc0a177475 +956, 0x4a8abfd1702ecb7d +957, 0x53f22886694736a7 +958, 0x8fc982597ced3e3 +959, 0x1bc46090f820fff7 +960, 0x8bd31f965d02229f +961, 0x65cd0cb29996ee53 +962, 0x702e0f4fcf8c2e9f +963, 0x293b77bff307a9a0 +964, 0x125a986b8b305788 +965, 0x416b0eea428ebf3c +966, 0xeac85421ab0e8469 +967, 0x7f5496095019aa68 +968, 0x1a96d7afbc708e0 +969, 0xb91262e6766e01e1 +970, 0xd0a549cc4ccc6954 +971, 0x75a9a073f50c8a0d +972, 0xae275d2c1c6cd23c +973, 0xcf159b5ec5d28fd4 +974, 0x75d0838ce9b92b +975, 0xd4eddcee6dc4677f +976, 0x6a0a8ad5df6b75b8 +977, 0x6f3fd0ef0f13ecc4 +978, 0xb75a5826c1a8f8a8 +979, 0xd47098bbc7943766 +980, 0x3d4ddd62d5f23dd1 +981, 0x760a904e4583841c +982, 0x2afeb5022b4cf1f +983, 0x66d5f653729f0a13 +984, 0x9a6a5ab62980d30f +985, 0xc332f5643bbf8d5b +986, 0x848fb702e4056a90 +987, 0xa057beaf3f9e8c5f +988, 0x6cc603e4560a6c6a +989, 0xec761811a7b23211 +990, 0xb14aa4090a82aaa5 +991, 0xe29d9d028a5b2dbb +992, 0x5564e53738d68f97 +993, 0xfabca36542eaaf3b +994, 0xb9912fcb782020a2 +995, 0xe865e01b349284fd +996, 0x540b5ff11c5f9274 +997, 0x3463f64e1e7451dc +998, 0xe15d3e2f33b735f8 +999, 0xf5433336eadef6e diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/data/sfc64-testset-2.csv b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/data/sfc64-testset-2.csv new file mode 100644 index 0000000000000000000000000000000000000000..70aebd5d539256e0abd453bb45828b691c6abdf0 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/data/sfc64-testset-2.csv @@ -0,0 +1,1001 @@ +seed, 0x0 +0, 0x91959e5fb96a6332 +1, 0x3c1dd8a25a7e9f21 +2, 0x657bdffc99798d9e +3, 0x1a04de320b19e022 +4, 0x65b92af0e5f3c61c +5, 0x9c84070ce8f743c0 +6, 0xbb10e573693cdb25 +7, 0xd65ea9e76b37fb6b +8, 0x503efd0e76c8ae66 +9, 0xd711dcd04c26d0f +10, 0x12f53f435814ac8c +11, 0xb392cd402cfc82bd +12, 0x461764550e06c889 +13, 0x716a48b3514e6979 +14, 0xdd0a322213c18ad7 +15, 0x6673a8ca0a05c4d7 +16, 0x2992ef333437f844 +17, 0xc4aaf7e8240b2aad +18, 0x6ab0a1af1f41474f +19, 0xb0bae400c226941d +20, 0xe5f80c2eeeab48c6 +21, 0x3832c6a93a4024bf +22, 0x280bd824fabe8368 +23, 0x66b626228321e5ff +24, 0xe0bdfba5325a307e +25, 0x3a5f65c6ef254e05 +26, 0x99ea12503cb02f94 +27, 0x5d01fd2db77d420b +28, 0x6959bf5f36b2368d +29, 0xd856e30c62b5f5be +30, 0xe33233e1d8140e66 +31, 0xb78be619d415fa8d +32, 0x4f943bb2cc63d3b +33, 0x9b1460b290952d81 +34, 0x19205d794826740e +35, 0x64617bd9d7a6a1ff +36, 0x30442124b55ea76a +37, 0xebbbc3b29d0333fc +38, 0x39235a0fe359751c +39, 0xf9629768891121aa +40, 0x32052f53f366e05a +41, 0x60cc5b412c925bc8 +42, 0xf8b7ecda1c0e5a9 +43, 0x195f036e170a2568 +44, 0xfe06d0381a9ca782 +45, 0x919d89e8b88eebbf +46, 0xa47fb30148cf0d43 +47, 0x5c983e99d5f9fd56 +48, 0xe7492cdb6a1d42cd +49, 0xf9cfe5c865b0cfd8 +50, 0x35b653367bbc3b99 +51, 0xb1d92f6f4d4e440b +52, 0x737e1d5bd87ed9c0 +53, 0x7a880ca1498f8e17 +54, 0x687dae8494f9a3f7 +55, 0x6bae1989f441d5d7 +56, 0x71ad3fa5a9195c2e +57, 0x16b3969779f5d03 +58, 0xd1bce2ac973f15b3 +59, 0xa114b1ee2ce0dcdd +60, 0x270d75c11eb1b8d5 +61, 0xc48ffa087c0a7bc +62, 0xaaf9dc48cda9848d +63, 0x8111cf10ef6e584d +64, 0x6736df6af40ee6f4 +65, 0x1a1a111682fbf98d +66, 0xeb217658e1cb3b5d +67, 0xcaf58a8b79de9dec +68, 0x25d0ffd63c88d7a1 +69, 0x4c498cd871b7f176 +70, 0x4069a6156eb0cf3c +71, 0xdf012f12edcdd867 +72, 0x7734c0ac8edb1689 +73, 0xed6960ac53dbc245 +74, 0x305e20da8868c661 +75, 0x5f0c7a3719956f95 +76, 0x66842bbe3b28895 +77, 0xb608bc9a31eac410 +78, 0xfcb17d5529503abd +79, 0x829ae5cbc29b92ee +80, 0x17f2f0027bc24f3a +81, 0x435926c33d8f44cc +82, 0x3ab899327098dbec +83, 0xaf78573b27f8ead8 +84, 0xa8b334fabcf8dc60 +85, 0xcdf3b366a6a303db +86, 0x8da9379dd62b34c8 +87, 0xb0ba511955f264a7 +88, 0x9d72e21a644f961d +89, 0xfac28382e2e7e710 +90, 0xd457065f048410aa +91, 0x1cae57d952563969 +92, 0x5a160a6223253e03 +93, 0x2c45df736d73c8bd +94, 0x7f651ebc6ad9cec5 +95, 0x77a6be96c7d2e7e7 +96, 0x1721fb1dbfd6546a +97, 0xf73f433ecff3c997 +98, 0xed1e80f680965bfe +99, 0x6705ad67a3003b30 +100, 0xac21134efcadb9f7 +101, 0x4d2ba0a91d456ac +102, 0x59da7b59434eb52b +103, 0x26c1d070fd414b5f +104, 0xed7079ddfce83d9a +105, 0x9277d21f88e0fb7a +106, 0xfae16b9a8d53d282 +107, 0xb08a0e2e405fdf7d +108, 0x2ea20df44229d6ec +109, 0x80e4634cd3612825 +110, 0xbe62e8aeba8f8a1a +111, 0x4981209769c190fb +112, 0xcec96ef14c7e1f65 +113, 0x73fe4457b47e7b53 +114, 0x1d66300677315c31 +115, 0xe26821290498c4cc +116, 0xf6110248fd8fb1c5 +117, 0x30fd7fe32dbd8be3 +118, 0x534ec9b910a2bd72 +119, 0x8f9bfe878bbf7382 +120, 0x4f4eb5295c0c2193 +121, 0xdeb22f03a913be9e +122, 0x40f716f8e2a8886c +123, 0xc65007d0e386cdb1 +124, 0x9bdd26d92b143a14 +125, 0xf644b0b77ea44625 +126, 0x75f5a53f6b01993a +127, 0xfe803e347bf41010 +128, 0x594bff5fa17bc360 +129, 0x3551edfb450373c7 +130, 0x898f9dad433615db +131, 0x923d2406daa26d49 +132, 0x99e07faccbc33426 +133, 0x7389f9ff4470f807 +134, 0xdc2a25957c6df90b +135, 0x33c6d8965ef3053f +136, 0x51a8f07e838f1ab +137, 0x91c5db369380274f +138, 0xc37de65ac56b207e +139, 0xfcc6d2375dde7f14 +140, 0xa4e6418bff505958 +141, 0x4b8b9f78e46953c4 +142, 0x255ab2e0f93cf278 +143, 0xdf650717af3d96ef +144, 0x2caa21cba3aae2b2 +145, 0xce7e46c6f393daa4 +146, 0x1d5b3573f9997ac7 +147, 0x5280c556e850847d +148, 0x32edc31bef920ad7 +149, 0xefaa6b0b08cf2c6 +150, 0x5151c99d97b111c5 +151, 0x35ccf4bf53d17590 +152, 0xa210d7bd8697b385 +153, 0xa9419f95738fbe61 +154, 0xdeccf93a1a4fdc90 +155, 0xd0ea3365b18e7a05 +156, 0x84122df6dcd31b9a +157, 0x33040a2125cea5f5 +158, 0xfe18306a862f6d86 +159, 0xdb97c8392e5c4457 +160, 0xc3e0fa735e80e422 +161, 0x7d106ff36467a0c1 +162, 0xb9825eecc720a76d +163, 0x7fefc6f771647081 +164, 0xf5df3f5b3977bf13 +165, 0x18fb22736d36f1e0 +166, 0xadc4637b4953abfc +167, 0x174e66d3e17974bd +168, 0xf1614c51df4db5db +169, 0x6664ecde5717b293 +170, 0xd5bc5b6839265c26 +171, 0xf6ca9ce1af3f1832 +172, 0xca696789a9d506ea +173, 0x7399c246c8f9d53 +174, 0xadf49049626417e2 +175, 0xbcd84af37d09ab91 +176, 0xbb41c177f3a3fa45 +177, 0x592becc814d55302 +178, 0xa88b4e65f6cfe5f7 +179, 0xa0a55e34ff879426 +180, 0x3c2ea6aa725b42b7 +181, 0x65ac4a407b1f9521 +182, 0xde63d53f7e88b556 +183, 0x18bc76696d015f40 +184, 0xd1363f2cd4c116a8 +185, 0x2fe859be19a48e4a +186, 0x83d6099b1415e656 +187, 0x43f2cbc1a4ee6410 +188, 0xb2eca3d3421c533d +189, 0xc52b98ea3f031f5d +190, 0xfe57eb01da07e9d1 +191, 0xf9377883537a6031 +192, 0x364030c05dac7add +193, 0x6815cb06b35d4404 +194, 0xceae2d4ce31894be +195, 0xc602bcdf6062bf6a +196, 0xc8e4bd8dcc6062e3 +197, 0x9c29e87b92a1a791 +198, 0x41e626b871ca9651 +199, 0x325c3d1fb8efbcd8 +200, 0x7dbbacf8e3419fb3 +201, 0x3602e72516bb7319 +202, 0x537a008ebd94d24b +203, 0xda7714fc9d4d161d +204, 0x1c8c73700e1b621b +205, 0x2749b80937d6c939 +206, 0x76ee6abac5b14d33 +207, 0xf18d1e92cb6a8b5c +208, 0x6ce9579d9291c721 +209, 0x60523c745a40e58 +210, 0x637f837fcc901757 +211, 0x2ff71b19661dc5b3 +212, 0x393ab586326ad16f +213, 0xa0970ea30fe742b7 +214, 0x570222d7f27fe5ae +215, 0x3b5806d43fd38629 +216, 0x129a0ad7420180c5 +217, 0x1c4726355778d52c +218, 0x7c1459cf77656499 +219, 0xfe038a0932132069 +220, 0x4c4cc317a937483a +221, 0xa333d24067e926ba +222, 0x401d9b6ab37f6ef2 +223, 0x87ad0e491ebe4a2a +224, 0xfc02f312e72d121d +225, 0xfde715b3b99767b2 +226, 0xd111c342ba521c92 +227, 0x83b221b10879c617 +228, 0x6a1bf5c01fdf4277 +229, 0x166bfc0c3f5892ee +230, 0x4608d556d7c57856 +231, 0x8d786857c95ece49 +232, 0x2d357445a1aca4ac +233, 0x79620dae28ecd796 +234, 0x90e715dc0f2201c4 +235, 0x173b68b4c9f4b665 +236, 0x4e14d040ebac4eef +237, 0xbd25960b4b892e +238, 0x911a199db6f1989d +239, 0xfe822d7c601fd2e0 +240, 0x9b4c1d58d8223a69 +241, 0x907c1891283843b0 +242, 0xf4868bf54061c4b2 +243, 0x17f8cd1fc24efd85 +244, 0xd44253f9af14c3aa +245, 0x16d0da0cb911d43c +246, 0x3c6a46615828e79a +247, 0x498591c1138e11a5 +248, 0xcc0f26336d0d6141 +249, 0x4d3ebc873212309a +250, 0x16bad7792d5c2c6a +251, 0x474215a80b2bbd11 +252, 0x7159848abd8492fc +253, 0x359341c50973685f +254, 0x27512ee7bf784a4a +255, 0x45228ea080f70447 +256, 0x880cab616500d50e +257, 0x12fae93f9830d56e +258, 0x6744ee64348d9acd +259, 0x484dada28cd2a828 +260, 0x98491d0729e41863 +261, 0x2f15aac43c2863b0 +262, 0x5727a34d77a1da0f +263, 0xa435cebef6a62eed +264, 0xd211697d57b053b0 +265, 0x65aa757b68bd557 +266, 0xe3a1b7a2d8a3e06a +267, 0x2adf64e67252a7a9 +268, 0xadadcb75cadee276 +269, 0x7934bc57ac8d97bf +270, 0xccff0d0f412e0606 +271, 0x101a82aa3e8f3db9 +272, 0xb0f2498094b4575c +273, 0xba2561d9ef26ed8a +274, 0xfbcd1268fc3febe1 +275, 0x9aa10bb19eb152e0 +276, 0xf496217a601a6d72 +277, 0xe4be1e4f2fa91363 +278, 0x473a602bf3dd68eb +279, 0xfe8ed2a48c26f4b5 +280, 0x20e94b1a00159476 +281, 0x93e1cb1c6af86ec7 +282, 0x4fcba3898f7442ba +283, 0x5150c3a3d94891df +284, 0x91cfce6c85b033ea +285, 0x625e8a832a806491 +286, 0x28c97ba72e3ec0b2 +287, 0x8e172de217c71ea1 +288, 0x926b80216c732639 +289, 0x28b19431a649ae3d +290, 0x57c039a6e95a3795 +291, 0xfbc354182fe52718 +292, 0x819dfd7c7d534cef +293, 0xabb4093a619ed44f +294, 0xe785b7ac6f656745 +295, 0xb647b4588b2f942f +296, 0x64cf870a14c72d27 +297, 0x6d4a4a2a0ba9b37e +298, 0x78bfb0427d7ce6b0 +299, 0x8dcc72b8bfc79ac6 +300, 0x1c14d915d5e76c99 +301, 0xaf48ddea6f096d79 +302, 0x51b39b67aa130d8 +303, 0x1aeeb39d4def06de +304, 0xd678092ffedfdd27 +305, 0x8f54787f325111d3 +306, 0xf2ca2e827beaa6bc +307, 0x339d134099e98545 +308, 0x1f6a8a7b33942e43 +309, 0x952c8065dbef669a +310, 0xe066aeb6690147f7 +311, 0xed25aa92cf58ebb6 +312, 0x7601edce215ef521 +313, 0xed1c5b396abd9434 +314, 0x4fd1e407535de9d5 +315, 0xccc8315a0d4d1441 +316, 0x85753e250bb86976 +317, 0xf232e469378761c3 +318, 0x81d691b8e9aef3c6 +319, 0x224a2f9cab0ad0e +320, 0x978f3d3e50007f4e +321, 0xd3713e6a6c0cbe60 +322, 0xcce8f1eadd41f80d +323, 0x34bda028a97d469 +324, 0x90e242fdf0f59183 +325, 0x4d749754fbc5f092 +326, 0x4399f5b7851cc87b +327, 0xcb921a5f25f6c5d7 +328, 0x120bf5d0162101 +329, 0x1304cc2aa352735a +330, 0xf7236c5d0d5d417b +331, 0xc31b320fc1654306 +332, 0xb468c6b23f3fb4e7 +333, 0xb5985b5bfaca4166 +334, 0x898285a1cd2f8375 +335, 0xa13493da372aa7c9 +336, 0x15c80c09c12634e7 +337, 0x9b765c5cc9d438bd +338, 0xee7da816a9201dcb +339, 0x92e269f73b5a248e +340, 0xa8086c5de81400ce +341, 0xe0053901853d42be +342, 0x821df32c012f433e +343, 0x17a6d69ca37387c7 +344, 0x2b10044bfba3501f +345, 0x8dfd262afc2e8515 +346, 0xd68c2c7b60226371 +347, 0xe81ac114e4416774 +348, 0x5896d60061ebc471 +349, 0xa996e3147811dbd1 +350, 0xa819c7b80ecb3661 +351, 0x982ad71b38afbc01 +352, 0xab152b65aa17b7fe +353, 0x4582bc282ef187ef +354, 0xab5a17fe8d9bc669 +355, 0x83664fa9cb0284b7 +356, 0x234c4b0091968f52 +357, 0x8ab5f51805688d37 +358, 0xe9e11186e0c53eda +359, 0x10df37ef1de2eccf +360, 0x780f1b0d52db968f +361, 0x50bd4ff292872cd5 +362, 0x51e681c265f5ad0 +363, 0x842c49660a527566 +364, 0x6e56ee026e9eda87 +365, 0x4cf39e40d8c80393 +366, 0x13e466df371f7e1f +367, 0xf2ce1799f38e028e +368, 0x833c8db7adc6ff0e +369, 0xc6e189abc2ec98f +370, 0xafebb3721283fec5 +371, 0xb49bc1eb5cc17bdc +372, 0xf1d02e818f5e4488 +373, 0xe5e9d5b41a1dd815 +374, 0xce8aca6573b1bfe5 +375, 0x9b0a5d70e268b1d5 +376, 0xf3c0503a8358f4de +377, 0x2681605dd755669d +378, 0xea265ca7601efc70 +379, 0xa93747f0a159439f +380, 0x62a86ede78a23e50 +381, 0xac8a18935c3d063c +382, 0x729c0a298f5059f5 +383, 0xbbf195e5b54399f4 +384, 0x38aa9d551f968900 +385, 0x3b3e700c58778caa +386, 0x68e6e33c4443957a +387, 0x7c56fc13eb269815 +388, 0xaf7daca39711804a +389, 0x50fde6d10f9544b3 +390, 0xf3d37159f6f6c03d +391, 0x82d298f5c1a71685 +392, 0x478661ac54c5002c +393, 0x6053768e1a324ae0 +394, 0xde8fb4a7e56707ea +395, 0xaa2809301faa8cf4 +396, 0x690a8d49fedd0722 +397, 0xe17c481b9c217de9 +398, 0x60d1d8a2b57288e3 +399, 0x149adfaadc6b0886 +400, 0xa3c18b6eb79cd5fa +401, 0x5774e3a091af5f58 +402, 0x2acca57ff30e5712 +403, 0x94454d67367c4b0c +404, 0x581b2985ac2df5ca +405, 0x71618e50744f3e70 +406, 0x270a7f3bd9a94ae6 +407, 0x3ef81af9bb36cd7b +408, 0x8a4a2592875254aa +409, 0x704ac6086fbb414a +410, 0xda774d5d3f57414d +411, 0xe20d3358b918ae9e +412, 0x934a6b9f7b91e247 +413, 0xf91649cde87ec42c +414, 0x248cec5f9b6ced30 +415, 0x56791809fd8d64ba +416, 0xf502b2765c1395f +417, 0x6b04ec973d75aa7f +418, 0xb0339f2794bb26f +419, 0x4c524636efbaea49 +420, 0x6bbf3876e9738748 +421, 0xf686524e754e9e24 +422, 0x8dafa05a42d19cd3 +423, 0xc5f069ab2434008e +424, 0x4fd64cc713cba76 +425, 0xdbf93450c881ed5f +426, 0x492e278ebabb59a2 +427, 0x993fddfde4542642 +428, 0xecde68a72c8d4e52 +429, 0xe0760b3074c311fd +430, 0x68dc0e7e06528707 +431, 0x52b50edf49c0fdc7 +432, 0xb2bd4185c138f412 +433, 0x431496d7e1d86f3 +434, 0xa4e605b037e26c44 +435, 0x58236ae1f0aca2b5 +436, 0x26c72c420fc314d8 +437, 0x20134e982ab99a2b +438, 0x544b59b8b211374b +439, 0x1301c42f3a14d993 +440, 0x52a6ea740f763b0f +441, 0xf209d70c2bebf119 +442, 0xac66a4ebc2aa1be +443, 0x683713ed35878788 +444, 0x2b5578acec06b80c +445, 0x86428efa11c45b36 +446, 0xb49010adb17d291e +447, 0x73b686bd8664b6be +448, 0x6d28ebf57b6884cc +449, 0x9712091230ff58d9 +450, 0xc9c91f74c38b286 +451, 0x776310ac41dc008e +452, 0x2f3739df0bf6a88e +453, 0x5792dc62b94db675 +454, 0x5715910d024b06af +455, 0xeb1dd745458da08 +456, 0xfce7b07ccfa851a7 +457, 0xc305f1e983ac368 +458, 0x485aa9519ac00bb0 +459, 0xa5354f6589fb0ea0 +460, 0x32fee02dfdbf4454 +461, 0x4d1ddc304bbefaaa +462, 0x789a270a1737e57e +463, 0x9f3072f4b1ed8156 +464, 0x4de3c00e89058120 +465, 0xb00a02529e0a86fa +466, 0x539f6f0edd845d9a +467, 0x85e578fe15a8c001 +468, 0xa12c8e1a72cce7d8 +469, 0xc6908abbc2b1828 +470, 0xcf70090774cbb38c +471, 0x3b636a6977b45d4a +472, 0xf0a731b220680b57 +473, 0x18973929f51443a8 +474, 0xe93e1fbe7eadabe +475, 0x8233730f0a6dfa02 +476, 0x66e50b6919b0ab74 +477, 0xb1aba87c97fd08a2 +478, 0xd4dffc1fbc117ad6 +479, 0x6f7fa65724b96e6a +480, 0x4bd5800dee92e0fa +481, 0xe18a959db6256da +482, 0xe53a291bc66df487 +483, 0xb7ec306a08651806 +484, 0x1847a6b80d2821e1 +485, 0xda50391283b14d39 +486, 0xacc4d3cd7cceb97a +487, 0x57f70185165b7bc6 +488, 0x302b6d597c3aaba7 +489, 0xa47f32d037eab51e +490, 0xe1509b4408abc559 +491, 0x4f30a1d7c2934157 +492, 0x2ad03e6c60b650b2 +493, 0x334d9c337b0a9064 +494, 0xc7f442821e7aac12 +495, 0xbcdeb09298694cdd +496, 0xe42402389f8f0fb4 +497, 0xe5de56af539df727 +498, 0x7017f9b2101ee240 +499, 0x1ee5e68d5b10001d +500, 0x436229051836387a +501, 0xcd532d6d6ec38fb7 +502, 0x30a66606fdf38272 +503, 0xfdaa2ab9cf798496 +504, 0x4277b4adec70e7df +505, 0x72cfc30256e0eaef +506, 0x3c3359fd9bd34917 +507, 0xb7aa89598856efb0 +508, 0xf72226f8bf299ef5 +509, 0x258c499275a4356f +510, 0x999a56bfc7f20d76 +511, 0x2b3e7432e20c18b +512, 0x2d1251332f760cb5 +513, 0x7420e0eea62157c5 +514, 0xe85c895aa27cec3d +515, 0x27a0545c7020d57c +516, 0xc68638a65b4fff0d +517, 0xfda473983a4ea747 +518, 0xd19fe65fb4c06062 +519, 0x6b1374e050ee15e4 +520, 0x80065ecd49bc4bef +521, 0x4ee655954bc838de +522, 0xe8fb777504a72299 +523, 0x86b652ea70f4bdde +524, 0xcdc9e0fbde7e4f33 +525, 0x352c0a50cd3ac56 +526, 0x4b8605d368be75dc +527, 0x1ac9ea8129efbc37 +528, 0x470325faa99f39c5 +529, 0x25dd7ef9adccf7a1 +530, 0x5ae2c7a03e965816 +531, 0xf733d2df59dacc7d +532, 0xa05bbf0a8a1a7a70 +533, 0xe8aa3f102846ef5f +534, 0xc9b85ec49ae71789 +535, 0xb904c14ed1cb1936 +536, 0x5ae618230b5f0444 +537, 0x97987fe47b5d7467 +538, 0xabb3aca8865ca761 +539, 0x38bfdf29d4508228 +540, 0x353654f408353330 +541, 0xeb7e92930ae4ef0d +542, 0xec50f1a7ca526b96 +543, 0xd5e2dc08b5697544 +544, 0x24c7fd69d5ec32df +545, 0x6f7e1095568b8620 +546, 0x6ed9c16ca13b3c8 +547, 0xe676ef460002130f +548, 0xa3a01a3992c4b430 +549, 0xe2130406c3b1f202 +550, 0xa8f7263e2aedcd20 +551, 0xc45d71ef2e35f507 +552, 0x37155594021da7ba +553, 0x22dc94f19de73159 +554, 0x7969fc6bffc5443f +555, 0x97def7e44faa6bfe +556, 0x8b940f5e8931d71f +557, 0xd95b1dd3f1a3fdd5 +558, 0x1c83bfdca615701a +559, 0xb7fcb56279ceca6b +560, 0xd84f8950f20dcd0 +561, 0xb03343698de3cbe0 +562, 0xf64565d448d71f71 +563, 0xda52b4676e0ae662 +564, 0xda39c2c05b4ffb91 +565, 0xb35e2560421f6a85 +566, 0x1a7b108d48ac3646 +567, 0xc4e264dc390d79ed +568, 0xa10727dfd9813256 +569, 0x40d23154e720e4f7 +570, 0xd9fa7cd7e313e119 +571, 0xcbf29107859e6013 +572, 0xc357338553d940b7 +573, 0x2641b7ab0bdfcbaa +574, 0xd12f2b6060533ae7 +575, 0xd0435aa626411c56 +576, 0x44af4a488a9cec72 +577, 0xb934232ea8fa5696 +578, 0x760a8b12072b572d +579, 0xfab18f9942cfa9b3 +580, 0x5676834c1fe84d16 +581, 0x9c54e4fddb353236 +582, 0xab49edfc9551f293 +583, 0x567f1fb45a871d +584, 0x32a967c873998834 +585, 0x99240aad380ef8d1 +586, 0x7f66cbd432859a64 +587, 0x4cdc8a4658166822 +588, 0x984e3984a5766492 +589, 0xa3b2d0a3d64d3d94 +590, 0x177f667172f2affc +591, 0xb1a90607a73a303f +592, 0xe600b6c36427f878 +593, 0xf758f9834cb7f466 +594, 0x8ee9fce4a3f36449 +595, 0xcb8f11533e7da347 +596, 0xe7cf647794dabd7c +597, 0xc9d92cfe6110806 +598, 0xea1335fa9145a1ec +599, 0xbc6c29821d094552 +600, 0x37b9d6a858cc8bc3 +601, 0xf24e4c694929893e +602, 0x55d025ce2d7d0004 +603, 0xccdc69acccf4267b +604, 0xc491c04340c222eb +605, 0xba50f75ecec9befb +606, 0x1ec7bd85b8fe3bb9 +607, 0xe4de66498c59ae8a +608, 0x38aa9e912712c889 +609, 0xcee0e43c5cc31566 +610, 0x72b69aa708fc7ed +611, 0xdff70b7f6fa96679 +612, 0xd6d71d82112aadc3 +613, 0x365177892cb78531 +614, 0xa54852b39de4f72c +615, 0x11dd5832bf16dd59 +616, 0x248a0f3369c97097 +617, 0xa14cec0260e26792 +618, 0x3517616ff142bed1 +619, 0x9b693ad39dab7636 +620, 0x739dff825e994434 +621, 0x67711e7356098c9 +622, 0xa81f8515d2fdf458 +623, 0xdac2908113fe568e +624, 0xe99944ebc6e2806a +625, 0x671728ca5b030975 +626, 0xfdad20edb2b4a789 +627, 0xedc6e466bd0369d2 +628, 0x88b5d469821f7e1b +629, 0x2eabf94049a522a5 +630, 0x247794b7a2f5a8e3 +631, 0x278942bdbe02c649 +632, 0xbe5a9a9196ab99c1 +633, 0x75955060866da1b5 +634, 0xdedcfa149273c0b5 +635, 0xdbeb7a57758f3867 +636, 0x7b9053347a2c8d5a +637, 0xa059b3f2eed338a5 +638, 0x59401a46ded3b79f +639, 0x38044ba56a6d19fb +640, 0x72c7221b4e77e779 +641, 0x526df3491a3a34da +642, 0xc3b31184ba16c0c2 +643, 0xd94c7144488624af +644, 0xcf966ee4dc373f91 +645, 0x62049e65dd416266 +646, 0x7c2adccb925bf8f +647, 0xd5fa5c22ed4ef8e1 +648, 0xd00134ebd11f2cd1 +649, 0xfbdf81767bed3634 +650, 0x62e8cc8ff66b6e26 +651, 0x3a72d6bcd4f2dcf7 +652, 0xf1cd45b1b46a86ed +653, 0x1271f98e0938bb9a +654, 0x82e6927e83dc31fa +655, 0x7b9b0e0acb67b92d +656, 0x6df503e397b2e701 +657, 0x93888f6fb561e0c3 +658, 0x393fb6069a40291 +659, 0x967a7d894cc0754d +660, 0x6e298996ad866333 +661, 0x5ff3cf5559d6ab46 +662, 0xd0d70508c40349f5 +663, 0xc64c66c0dd426b33 +664, 0x8fea340ee35c64dd +665, 0xf9cd381eb3060005 +666, 0xfcc37c2799fc0b11 +667, 0x6a37c91d65b489fa +668, 0x57231000fa0a0c9d +669, 0x55f6e292c6703f9a +670, 0xd0508ffbfa55a7a6 +671, 0x885db543276bdac8 +672, 0xc26dbe6a26b0e704 +673, 0x21f884874ebd709e +674, 0x711f0b6c8f732220 +675, 0x354d0a361eaee195 +676, 0x721344d8d30b006a +677, 0xa0e090a0d3a56f07 +678, 0x16b3d5d823a4952b +679, 0x59d7874bc9eae7b6 +680, 0x9bbb32710076455f +681, 0xd4fb22242ffabafd +682, 0xe1d4ac6770be1d89 +683, 0xb259cedebc73dc8a +684, 0x35faaa3b4246ab69 +685, 0x5d26addefdaee89 +686, 0x8e7ec350da0f3545 +687, 0xd0f316eed9f8fc79 +688, 0x98b2a52c9bf291b2 +689, 0xe4d294a8aca6a314 +690, 0x25bd554e6aa7673c +691, 0xcfde5dcba5be2a6c +692, 0xb5e01fb48d2d2107 +693, 0xe1caf28948028536 +694, 0xd434aa0a26f3ee9b +695, 0xd17723381641b8f6 +696, 0xfe73bd1f3f3768a2 +697, 0x1cc6b1abd08d67e9 +698, 0x247e328371a28de0 +699, 0x502e7942e5a9104a +700, 0x6a030fd242eb4502 +701, 0xa2ffe02744014ce8 +702, 0x59290763b18fe04e +703, 0xcf14241564271436 +704, 0xb0fb73c3c1503aff +705, 0x94e27c622f82137a +706, 0x747a5b406ac3e1f0 +707, 0x9a914e96a732031d +708, 0x59f68c6c8f078835 +709, 0x809d012c73eb4724 +710, 0x5b3c3b73e1b37d74 +711, 0xdde60ef3ba49cdf7 +712, 0x87a14e1f9c761986 +713, 0x4109b960604522af +714, 0x122d0e1ed0eb6bb9 +715, 0xadc0d29e80bfe33 +716, 0xa25b1b44f5fc8e4e +717, 0xbab85d8a9b793f20 +718, 0x825f4cbced0e7d1e +719, 0x2d6ae8807acb37ea +720, 0x8234420adce2e39 +721, 0x4a8ad4da6b804807 +722, 0x1e19f9bc215e5245 +723, 0x1d6f4848a916dd5e +724, 0x9ac40dfcdc2d39cc +725, 0x9f3524e3086155ec +726, 0x861fffc43124b2ef +727, 0xe640e3b756396372 +728, 0x41cb0f0c5e149669 +729, 0xe0bd37e1192e4205 +730, 0x62917d3858f4ce47 +731, 0xa36e7eb4d855820a +732, 0x204b90255a3bf724 +733, 0x66ee83a0175535bc +734, 0x2c14ce7c6b0c1423 +735, 0x85d9495fa514f70d +736, 0x5a4fe45ead874dbc +737, 0xe72248dcb8cfc863 +738, 0xfc21ff2932ed98cd +739, 0xcbba1edd735b5cad +740, 0x91ddc32809679bf5 +741, 0x192cdf2c7631ea1f +742, 0xbbc451ddf2ea286f +743, 0xad9e80cae2397a64 +744, 0x6918f0119b95d0e5 +745, 0xa40379017a27d70a +746, 0x1aaeddb600e61e1 +747, 0x15afd93cbd7adda9 +748, 0x156719bc2b757ff4 +749, 0x13d9a59e2b2df49d +750, 0x9a490986eaddf0a +751, 0xef9a350f0b3eb6b4 +752, 0x5de7f6295ba4fa4d +753, 0x7f37fd087c3fdb49 +754, 0xa9fe3749d6f3f209 +755, 0x50912ac036d9bfb +756, 0x982cb4d726a441f8 +757, 0x8ca8d8af59b872d0 +758, 0x7f8adfb0ceeade8a +759, 0xdad390ec742be44 +760, 0xa637944d0045be5b +761, 0x3569a3b3af807061 +762, 0x9599da8eae14511d +763, 0xc333e8d19589b01a +764, 0xfb9b524a20b571e1 +765, 0xbd9dc8b37ce5c3e1 +766, 0x142333005fa389ac +767, 0x1368bc37cd5bcce1 +768, 0x16094907ad6ecf73 +769, 0xb32c90dbba4c1130 +770, 0x82761d97c1747dd0 +771, 0x599f9f267ae3444d +772, 0x79ad3382994852e1 +773, 0x2511f06d9ef06e54 +774, 0xb35e6ab7d5bbddae +775, 0xfca9fa83a2988732 +776, 0x7d4350f0394ac3ba +777, 0xa52a9527bb176ea3 +778, 0xb49fa0ceb2aa8353 +779, 0x1f62e504d1468cc0 +780, 0xe1a77bfccce6efc3 +781, 0x776cdff4dc0d6797 +782, 0x56612e39b652c1f2 +783, 0x5f096a29294eda04 +784, 0x7978abc3aabd8b23 +785, 0x79dd875e0485b979 +786, 0x8a98aa4d5735d778 +787, 0xcca43940f69d2388 +788, 0xb2d4b156f144f93a +789, 0xbd528a676e9a862 +790, 0x2a394939c8e7ec5e +791, 0xb1da900c6efe4abc +792, 0x9869af479de4c034 +793, 0x78dbdfb88ac7c1db +794, 0x18cb169143088041 +795, 0xe69e5461c51a3e13 +796, 0x5389fa16ea98183c +797, 0xed7c80d1be1ea520 +798, 0x87246fc359758ced +799, 0xab323eba95fae4ed +800, 0xbc4c0dde7f8a1828 +801, 0xdb739f7955610b1a +802, 0xecd8c68c3434cc +803, 0x138c2eb88c477f44 +804, 0x28a65f96727aae41 +805, 0xdee879f2cf5629d +806, 0x684f0c90ef20070f +807, 0xa24a819ef5621800 +808, 0x8d0054f870e4fdcb +809, 0x99e8c6e695b600b +810, 0x50b705245891f7c3 +811, 0xc02eed3a6e58e51a +812, 0x443d64e95443606c +813, 0xca24959cfbd2d120 +814, 0xe072609ea48815bc +815, 0xbcc715026590315b +816, 0x3e76df24d7aa5938 +817, 0xd8ff04940d9b79ae +818, 0x54474ce790059bcd +819, 0x278390dd6aa70e81 +820, 0xf4df619fe35414e4 +821, 0x757d71270264e615 +822, 0x1e8a373699c11b23 +823, 0xef68c82046e67dd6 +824, 0xe280006599972620 +825, 0x234e095183b0f4d6 +826, 0xe3b7560ed9839749 +827, 0xcd5ec4086572332e +828, 0xc41c0d4aaa279108 +829, 0x4b9cd6126bc16a6d +830, 0x4a7252734f3e3dd0 +831, 0xb3132df156cc103a +832, 0xf9e4abbf7b64464a +833, 0xf936df27fb3c47b7 +834, 0x9142960873f6d71a +835, 0x4ba6aa3235cdb10d +836, 0x3237a2e765ba7766 +837, 0xd62f0b94c8e99e54 +838, 0x26b682f90a3ae41b +839, 0x40ad5e82072b6f81 +840, 0xd0198101f5484000 +841, 0xe4fac60ba11c332 +842, 0x472d0b0a95ef9d38 +843, 0x8512557aec5a3d8f +844, 0xef83169d3efd4de9 +845, 0x53fe89283e7a7676 +846, 0x2f50933053d69fc4 +847, 0x76f5e4362e2e53a2 +848, 0x8676fdccce28874a +849, 0x2737764c1fb1f821 +850, 0x4a6f70afc066ab55 +851, 0x27f8e151e310fca4 +852, 0xd606960ccbe85161 +853, 0xcce51d7ddd270a32 +854, 0xb4235999794875c2 +855, 0x580084e358e884 +856, 0x2159d5e6dc8586d7 +857, 0x87bd54d8599b3ba4 +858, 0x3e9ade6a2181664 +859, 0x5e6e140406d97623 +860, 0x511545d5aa0080a2 +861, 0xf49d78ed219aac57 +862, 0xbece1f9c90b8ea87 +863, 0x1c741cac36a2c514 +864, 0x7453c141047db967 +865, 0xd751832a5037eba2 +866, 0x71370a3f30ada1f7 +867, 0x7c01cf2dcb408631 +868, 0x1052a4fbdccc0fa1 +869, 0x13d525c9df3fb6c +870, 0xa3aa8dbfee760c55 +871, 0xc0288d200f5155cf +872, 0x79f4bcd12af567c3 +873, 0x8160d163bb548755 +874, 0x5cf2995fb69fd2df +875, 0xcc98ed01396639df +876, 0xad95f1d9cfc8256e +877, 0xa3df27d9fbdbfb9d +878, 0x83e5f5dda4d52929 +879, 0x9adc05043009f55b +880, 0xdfe8329dfde1c001 +881, 0x9980ccdd5298e6a2 +882, 0x636a7bd134f6ef56 +883, 0xef5ff780c4be6ba4 +884, 0x290d71dc77a56d16 +885, 0x6d65db9ff58de1e6 +886, 0x944b063b3805a696 +887, 0xce468ca2cce33008 +888, 0x5ba1ccb840f80f48 +889, 0x28ddce36fc9ad268 +890, 0x4f77ef254d507a21 +891, 0xce9b4057fadf3ab +892, 0xb518bc68298730e6 +893, 0xd2eb5b8e2ec665b0 +894, 0xe1583303a4f87344 +895, 0x9d5a0df4fbe1bed5 +896, 0x2ba9bc03ec8cfd07 +897, 0x479ed880a96ca669 +898, 0xcedf96338324771a +899, 0x312f4fc2da41ffaa +900, 0xa0eb9cf23b5e1ed8 +901, 0xf8f88f975dc3f539 +902, 0x4a37e185d0e96e0f +903, 0xf829654a5c0b46f9 +904, 0x3909cca7a7f8c7fb +905, 0x4c2e1d66ceb45105 +906, 0xaffaa19e1db8af87 +907, 0x9ec498246bd18c76 +908, 0x21d51558edc089da +909, 0xe8984112cd1b1561 +910, 0x7de1d2cf54b0c0e1 +911, 0xa06729aed50bfb9d +912, 0xcf19f733e5db19e1 +913, 0x70edf2624ab777cd +914, 0x46685becad10e078 +915, 0x825e0f6add46785 +916, 0x66d4af3b15f70de4 +917, 0xc676614b0666b21 +918, 0x282a916c864f5cb7 +919, 0x2707283a3f512167 +920, 0x37ff3afda7461623 +921, 0xc767eb1205e4ca86 +922, 0x46b359aecc4ea25b +923, 0x67fbbb797a16dbb1 +924, 0x64fd4ba57122290e +925, 0x8acc2a8ae59d8fac +926, 0x64a49298599acc67 +927, 0xedf00de67177ce30 +928, 0x1ea9d8d7e76d2d2c +929, 0x363fcac323f70eb2 +930, 0x19e6e3ec8a9712eb +931, 0xca541e96b0961f09 +932, 0x4d8fd34c2822ec46 +933, 0x2fdd56a50b32f705 +934, 0xaac2fcf251e3fd3 +935, 0xb0c600299e57045c +936, 0xd951ec589e909e38 +937, 0x4dc8414390cae508 +938, 0x537ef9d5e2321344 +939, 0xa57bc21fd31aa2dc +940, 0xa3a60df564183750 +941, 0xbe69a5ce2e369fb6 +942, 0x7744601f4c053ec8 +943, 0x3838452af42f2612 +944, 0xd4f0dad7115a54e9 +945, 0x629cf68d8009a624 +946, 0x2211c8fa34cb98cb +947, 0x8040b19e2213db83 +948, 0xb2a86d3ba2384fd +949, 0x4b85cec4f93f0dab +950, 0xc8d212d21ea6845d +951, 0x5b271a03a4fe2be0 +952, 0xff4f671319ad8434 +953, 0x8e615a919d5afa96 +954, 0xea7f47c53161160a +955, 0x33273930b13c6efc +956, 0x98eedda27fb59c3c +957, 0x188dc5e92e939677 +958, 0x9dbd0fa0911430f1 +959, 0x5b3dcf3fa75dfd2b +960, 0x3f03846febdb275d +961, 0x20cc24faea9e9cf6 +962, 0x854f3ac66199ff5d +963, 0x31169ac99d341e6f +964, 0xa85daed3c0bc1bbe +965, 0x64633711e71ba5dd +966, 0x530e79978dc73334 +967, 0x636f2ee6e20aef13 +968, 0xf6220f8b6d9a58fb +969, 0x425db8fa32141a7b +970, 0xac7c210f4b02be95 +971, 0x5fe8cfbe197a7754 +972, 0xfff7d40c79420ea +973, 0x5f8bab9ef4697b77 +974, 0xaf6fe54e45b23fe8 +975, 0xce79456ccc70bbce +976, 0x645ef680f48f1c00 +977, 0xa4dfac46e2028595 +978, 0x6bece4c41effc5df +979, 0xd316df886442641f +980, 0xa4f6ff994edd2a6 +981, 0x30281ae3cc49abe4 +982, 0x39acb7b663dea974 +983, 0x5e8829b01a7c06fb +984, 0x87bdb08cf027f13e +985, 0xdfa5ede784e802f6 +986, 0x46d03d55711c38cc +987, 0xa55a961fc9788306 +988, 0xbf09ded495a2e57a +989, 0xcd601b29a639cc16 +990, 0x2193ce026bfd1085 +991, 0x25ba27f3f225be13 +992, 0x6f685be82f64f2fe +993, 0xec8454108229c450 +994, 0x6e79d8d205447a44 +995, 0x9ed7b6a96b9ccd68 +996, 0xae7134b3b7f8ee37 +997, 0x66963de0e5ebcc02 +998, 0x29c8dcd0d17c423f +999, 0xfb8482c827eb90bc diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/test_direct.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/test_direct.py new file mode 100644 index 0000000000000000000000000000000000000000..fa2ae866beeb7c36a585eaf9eb04df31a2f2a6c3 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/test_direct.py @@ -0,0 +1,518 @@ +import os +from os.path import join +import sys + +import numpy as np +from numpy.testing import (assert_equal, assert_allclose, assert_array_equal, + assert_raises) +import pytest + +from numpy.random import ( + Generator, MT19937, PCG64, PCG64DXSM, Philox, RandomState, SeedSequence, + SFC64, default_rng +) +from numpy.random._common import interface + +try: + import cffi # noqa: F401 + + MISSING_CFFI = False +except ImportError: + MISSING_CFFI = True + +try: + import ctypes # noqa: F401 + + MISSING_CTYPES = False +except ImportError: + MISSING_CTYPES = False + +if sys.flags.optimize > 1: + # no docstrings present to inspect when PYTHONOPTIMIZE/Py_OptimizeFlag > 1 + # cffi cannot succeed + MISSING_CFFI = True + + +pwd = os.path.dirname(os.path.abspath(__file__)) + + +def assert_state_equal(actual, target): + for key in actual: + if isinstance(actual[key], dict): + assert_state_equal(actual[key], target[key]) + elif isinstance(actual[key], np.ndarray): + assert_array_equal(actual[key], target[key]) + else: + assert actual[key] == target[key] + + +def uint32_to_float32(u): + return ((u >> np.uint32(8)) * (1.0 / 2**24)).astype(np.float32) + + +def uniform32_from_uint64(x): + x = np.uint64(x) + upper = np.array(x >> np.uint64(32), dtype=np.uint32) + lower = np.uint64(0xffffffff) + lower = np.array(x & lower, dtype=np.uint32) + joined = np.column_stack([lower, upper]).ravel() + return uint32_to_float32(joined) + + +def uniform32_from_uint53(x): + x = np.uint64(x) >> np.uint64(16) + x = np.uint32(x & np.uint64(0xffffffff)) + return uint32_to_float32(x) + + +def uniform32_from_uint32(x): + return uint32_to_float32(x) + + +def uniform32_from_uint(x, bits): + if bits == 64: + return uniform32_from_uint64(x) + elif bits == 53: + return uniform32_from_uint53(x) + elif bits == 32: + return uniform32_from_uint32(x) + else: + raise NotImplementedError + + +def uniform_from_uint(x, bits): + if bits in (64, 63, 53): + return uniform_from_uint64(x) + elif bits == 32: + return uniform_from_uint32(x) + + +def uniform_from_uint64(x): + return (x >> np.uint64(11)) * (1.0 / 9007199254740992.0) + + +def uniform_from_uint32(x): + out = np.empty(len(x) // 2) + for i in range(0, len(x), 2): + a = x[i] >> 5 + b = x[i + 1] >> 6 + out[i // 2] = (a * 67108864.0 + b) / 9007199254740992.0 + return out + + +def uniform_from_dsfmt(x): + return x.view(np.double) - 1.0 + + +def gauss_from_uint(x, n, bits): + if bits in (64, 63): + doubles = uniform_from_uint64(x) + elif bits == 32: + doubles = uniform_from_uint32(x) + else: # bits == 'dsfmt' + doubles = uniform_from_dsfmt(x) + gauss = [] + loc = 0 + x1 = x2 = 0.0 + while len(gauss) < n: + r2 = 2 + while r2 >= 1.0 or r2 == 0.0: + x1 = 2.0 * doubles[loc] - 1.0 + x2 = 2.0 * doubles[loc + 1] - 1.0 + r2 = x1 * x1 + x2 * x2 + loc += 2 + + f = np.sqrt(-2.0 * np.log(r2) / r2) + gauss.append(f * x2) + gauss.append(f * x1) + + return gauss[:n] + + +def test_seedsequence(): + from numpy.random.bit_generator import (ISeedSequence, + ISpawnableSeedSequence, + SeedlessSeedSequence) + + s1 = SeedSequence(range(10), spawn_key=(1, 2), pool_size=6) + s1.spawn(10) + s2 = SeedSequence(**s1.state) + assert_equal(s1.state, s2.state) + assert_equal(s1.n_children_spawned, s2.n_children_spawned) + + # The interfaces cannot be instantiated themselves. + assert_raises(TypeError, ISeedSequence) + assert_raises(TypeError, ISpawnableSeedSequence) + dummy = SeedlessSeedSequence() + assert_raises(NotImplementedError, dummy.generate_state, 10) + assert len(dummy.spawn(10)) == 10 + + +def test_generator_spawning(): + """ Test spawning new generators and bit_generators directly. + """ + rng = np.random.default_rng() + seq = rng.bit_generator.seed_seq + new_ss = seq.spawn(5) + expected_keys = [seq.spawn_key + (i,) for i in range(5)] + assert [c.spawn_key for c in new_ss] == expected_keys + + new_bgs = rng.bit_generator.spawn(5) + expected_keys = [seq.spawn_key + (i,) for i in range(5, 10)] + assert [bg.seed_seq.spawn_key for bg in new_bgs] == expected_keys + + new_rngs = rng.spawn(5) + expected_keys = [seq.spawn_key + (i,) for i in range(10, 15)] + found_keys = [rng.bit_generator.seed_seq.spawn_key for rng in new_rngs] + assert found_keys == expected_keys + + # Sanity check that streams are actually different: + assert new_rngs[0].uniform() != new_rngs[1].uniform() + + +def test_non_spawnable(): + from numpy.random.bit_generator import ISeedSequence + + class FakeSeedSequence: + def generate_state(self, n_words, dtype=np.uint32): + return np.zeros(n_words, dtype=dtype) + + ISeedSequence.register(FakeSeedSequence) + + rng = np.random.default_rng(FakeSeedSequence()) + + with pytest.raises(TypeError, match="The underlying SeedSequence"): + rng.spawn(5) + + with pytest.raises(TypeError, match="The underlying SeedSequence"): + rng.bit_generator.spawn(5) + + +class Base: + dtype = np.uint64 + data2 = data1 = {} + + @classmethod + def setup_class(cls): + cls.bit_generator = PCG64 + cls.bits = 64 + cls.dtype = np.uint64 + cls.seed_error_type = TypeError + cls.invalid_init_types = [] + cls.invalid_init_values = [] + + @classmethod + def _read_csv(cls, filename): + with open(filename) as csv: + seed = csv.readline() + seed = seed.split(',') + seed = [int(s.strip(), 0) for s in seed[1:]] + data = [] + for line in csv: + data.append(int(line.split(',')[-1].strip(), 0)) + return {'seed': seed, 'data': np.array(data, dtype=cls.dtype)} + + def test_raw(self): + bit_generator = self.bit_generator(*self.data1['seed']) + uints = bit_generator.random_raw(1000) + assert_equal(uints, self.data1['data']) + + bit_generator = self.bit_generator(*self.data1['seed']) + uints = bit_generator.random_raw() + assert_equal(uints, self.data1['data'][0]) + + bit_generator = self.bit_generator(*self.data2['seed']) + uints = bit_generator.random_raw(1000) + assert_equal(uints, self.data2['data']) + + def test_random_raw(self): + bit_generator = self.bit_generator(*self.data1['seed']) + uints = bit_generator.random_raw(output=False) + assert uints is None + uints = bit_generator.random_raw(1000, output=False) + assert uints is None + + def test_gauss_inv(self): + n = 25 + rs = RandomState(self.bit_generator(*self.data1['seed'])) + gauss = rs.standard_normal(n) + assert_allclose(gauss, + gauss_from_uint(self.data1['data'], n, self.bits)) + + rs = RandomState(self.bit_generator(*self.data2['seed'])) + gauss = rs.standard_normal(25) + assert_allclose(gauss, + gauss_from_uint(self.data2['data'], n, self.bits)) + + def test_uniform_double(self): + rs = Generator(self.bit_generator(*self.data1['seed'])) + vals = uniform_from_uint(self.data1['data'], self.bits) + uniforms = rs.random(len(vals)) + assert_allclose(uniforms, vals) + assert_equal(uniforms.dtype, np.float64) + + rs = Generator(self.bit_generator(*self.data2['seed'])) + vals = uniform_from_uint(self.data2['data'], self.bits) + uniforms = rs.random(len(vals)) + assert_allclose(uniforms, vals) + assert_equal(uniforms.dtype, np.float64) + + def test_uniform_float(self): + rs = Generator(self.bit_generator(*self.data1['seed'])) + vals = uniform32_from_uint(self.data1['data'], self.bits) + uniforms = rs.random(len(vals), dtype=np.float32) + assert_allclose(uniforms, vals) + assert_equal(uniforms.dtype, np.float32) + + rs = Generator(self.bit_generator(*self.data2['seed'])) + vals = uniform32_from_uint(self.data2['data'], self.bits) + uniforms = rs.random(len(vals), dtype=np.float32) + assert_allclose(uniforms, vals) + assert_equal(uniforms.dtype, np.float32) + + def test_repr(self): + rs = Generator(self.bit_generator(*self.data1['seed'])) + assert 'Generator' in repr(rs) + assert f'{id(rs):#x}'.upper().replace('X', 'x') in repr(rs) + + def test_str(self): + rs = Generator(self.bit_generator(*self.data1['seed'])) + assert 'Generator' in str(rs) + assert str(self.bit_generator.__name__) in str(rs) + assert f'{id(rs):#x}'.upper().replace('X', 'x') not in str(rs) + + def test_pickle(self): + import pickle + + bit_generator = self.bit_generator(*self.data1['seed']) + state = bit_generator.state + bitgen_pkl = pickle.dumps(bit_generator) + reloaded = pickle.loads(bitgen_pkl) + reloaded_state = reloaded.state + assert_array_equal(Generator(bit_generator).standard_normal(1000), + Generator(reloaded).standard_normal(1000)) + assert bit_generator is not reloaded + assert_state_equal(reloaded_state, state) + + ss = SeedSequence(100) + aa = pickle.loads(pickle.dumps(ss)) + assert_equal(ss.state, aa.state) + + def test_invalid_state_type(self): + bit_generator = self.bit_generator(*self.data1['seed']) + with pytest.raises(TypeError): + bit_generator.state = {'1'} + + def test_invalid_state_value(self): + bit_generator = self.bit_generator(*self.data1['seed']) + state = bit_generator.state + state['bit_generator'] = 'otherBitGenerator' + with pytest.raises(ValueError): + bit_generator.state = state + + def test_invalid_init_type(self): + bit_generator = self.bit_generator + for st in self.invalid_init_types: + with pytest.raises(TypeError): + bit_generator(*st) + + def test_invalid_init_values(self): + bit_generator = self.bit_generator + for st in self.invalid_init_values: + with pytest.raises((ValueError, OverflowError)): + bit_generator(*st) + + def test_benchmark(self): + bit_generator = self.bit_generator(*self.data1['seed']) + bit_generator._benchmark(1) + bit_generator._benchmark(1, 'double') + with pytest.raises(ValueError): + bit_generator._benchmark(1, 'int32') + + @pytest.mark.skipif(MISSING_CFFI, reason='cffi not available') + def test_cffi(self): + bit_generator = self.bit_generator(*self.data1['seed']) + cffi_interface = bit_generator.cffi + assert isinstance(cffi_interface, interface) + other_cffi_interface = bit_generator.cffi + assert other_cffi_interface is cffi_interface + + @pytest.mark.skipif(MISSING_CTYPES, reason='ctypes not available') + def test_ctypes(self): + bit_generator = self.bit_generator(*self.data1['seed']) + ctypes_interface = bit_generator.ctypes + assert isinstance(ctypes_interface, interface) + other_ctypes_interface = bit_generator.ctypes + assert other_ctypes_interface is ctypes_interface + + def test_getstate(self): + bit_generator = self.bit_generator(*self.data1['seed']) + state = bit_generator.state + alt_state = bit_generator.__getstate__() + assert_state_equal(state, alt_state) + + +class TestPhilox(Base): + @classmethod + def setup_class(cls): + cls.bit_generator = Philox + cls.bits = 64 + cls.dtype = np.uint64 + cls.data1 = cls._read_csv( + join(pwd, './data/philox-testset-1.csv')) + cls.data2 = cls._read_csv( + join(pwd, './data/philox-testset-2.csv')) + cls.seed_error_type = TypeError + cls.invalid_init_types = [] + cls.invalid_init_values = [(1, None, 1), (-1,), (None, None, 2 ** 257 + 1)] + + def test_set_key(self): + bit_generator = self.bit_generator(*self.data1['seed']) + state = bit_generator.state + keyed = self.bit_generator(counter=state['state']['counter'], + key=state['state']['key']) + assert_state_equal(bit_generator.state, keyed.state) + + +class TestPCG64(Base): + @classmethod + def setup_class(cls): + cls.bit_generator = PCG64 + cls.bits = 64 + cls.dtype = np.uint64 + cls.data1 = cls._read_csv(join(pwd, './data/pcg64-testset-1.csv')) + cls.data2 = cls._read_csv(join(pwd, './data/pcg64-testset-2.csv')) + cls.seed_error_type = (ValueError, TypeError) + cls.invalid_init_types = [(3.2,), ([None],), (1, None)] + cls.invalid_init_values = [(-1,)] + + def test_advance_symmetry(self): + rs = Generator(self.bit_generator(*self.data1['seed'])) + state = rs.bit_generator.state + step = -0x9e3779b97f4a7c150000000000000000 + rs.bit_generator.advance(step) + val_neg = rs.integers(10) + rs.bit_generator.state = state + rs.bit_generator.advance(2**128 + step) + val_pos = rs.integers(10) + rs.bit_generator.state = state + rs.bit_generator.advance(10 * 2**128 + step) + val_big = rs.integers(10) + assert val_neg == val_pos + assert val_big == val_pos + + def test_advange_large(self): + rs = Generator(self.bit_generator(38219308213743)) + pcg = rs.bit_generator + state = pcg.state["state"] + initial_state = 287608843259529770491897792873167516365 + assert state["state"] == initial_state + pcg.advance(sum(2**i for i in (96, 64, 32, 16, 8, 4, 2, 1))) + state = pcg.state["state"] + advanced_state = 135275564607035429730177404003164635391 + assert state["state"] == advanced_state + + +class TestPCG64DXSM(Base): + @classmethod + def setup_class(cls): + cls.bit_generator = PCG64DXSM + cls.bits = 64 + cls.dtype = np.uint64 + cls.data1 = cls._read_csv(join(pwd, './data/pcg64dxsm-testset-1.csv')) + cls.data2 = cls._read_csv(join(pwd, './data/pcg64dxsm-testset-2.csv')) + cls.seed_error_type = (ValueError, TypeError) + cls.invalid_init_types = [(3.2,), ([None],), (1, None)] + cls.invalid_init_values = [(-1,)] + + def test_advance_symmetry(self): + rs = Generator(self.bit_generator(*self.data1['seed'])) + state = rs.bit_generator.state + step = -0x9e3779b97f4a7c150000000000000000 + rs.bit_generator.advance(step) + val_neg = rs.integers(10) + rs.bit_generator.state = state + rs.bit_generator.advance(2**128 + step) + val_pos = rs.integers(10) + rs.bit_generator.state = state + rs.bit_generator.advance(10 * 2**128 + step) + val_big = rs.integers(10) + assert val_neg == val_pos + assert val_big == val_pos + + def test_advange_large(self): + rs = Generator(self.bit_generator(38219308213743)) + pcg = rs.bit_generator + state = pcg.state + initial_state = 287608843259529770491897792873167516365 + assert state["state"]["state"] == initial_state + pcg.advance(sum(2**i for i in (96, 64, 32, 16, 8, 4, 2, 1))) + state = pcg.state["state"] + advanced_state = 277778083536782149546677086420637664879 + assert state["state"] == advanced_state + + +class TestMT19937(Base): + @classmethod + def setup_class(cls): + cls.bit_generator = MT19937 + cls.bits = 32 + cls.dtype = np.uint32 + cls.data1 = cls._read_csv(join(pwd, './data/mt19937-testset-1.csv')) + cls.data2 = cls._read_csv(join(pwd, './data/mt19937-testset-2.csv')) + cls.seed_error_type = ValueError + cls.invalid_init_types = [] + cls.invalid_init_values = [(-1,)] + + def test_seed_float_array(self): + assert_raises(TypeError, self.bit_generator, np.array([np.pi])) + assert_raises(TypeError, self.bit_generator, np.array([-np.pi])) + assert_raises(TypeError, self.bit_generator, np.array([np.pi, -np.pi])) + assert_raises(TypeError, self.bit_generator, np.array([0, np.pi])) + assert_raises(TypeError, self.bit_generator, [np.pi]) + assert_raises(TypeError, self.bit_generator, [0, np.pi]) + + def test_state_tuple(self): + rs = Generator(self.bit_generator(*self.data1['seed'])) + bit_generator = rs.bit_generator + state = bit_generator.state + desired = rs.integers(2 ** 16) + tup = (state['bit_generator'], state['state']['key'], + state['state']['pos']) + bit_generator.state = tup + actual = rs.integers(2 ** 16) + assert_equal(actual, desired) + tup = tup + (0, 0.0) + bit_generator.state = tup + actual = rs.integers(2 ** 16) + assert_equal(actual, desired) + + +class TestSFC64(Base): + @classmethod + def setup_class(cls): + cls.bit_generator = SFC64 + cls.bits = 64 + cls.dtype = np.uint64 + cls.data1 = cls._read_csv( + join(pwd, './data/sfc64-testset-1.csv')) + cls.data2 = cls._read_csv( + join(pwd, './data/sfc64-testset-2.csv')) + cls.seed_error_type = (ValueError, TypeError) + cls.invalid_init_types = [(3.2,), ([None],), (1, None)] + cls.invalid_init_values = [(-1,)] + + +class TestDefaultRNG: + def test_seed(self): + for args in [(), (None,), (1234,), ([1234, 5678],)]: + rg = default_rng(*args) + assert isinstance(rg.bit_generator, PCG64) + + def test_passthrough(self): + bg = Philox() + rg = default_rng(bg) + assert rg.bit_generator is bg + rg2 = default_rng(rg) + assert rg2 is rg + assert rg2.bit_generator is bg diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/test_extending.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/test_extending.py new file mode 100644 index 0000000000000000000000000000000000000000..2783d1cdd9acd183261c30e50d7031f4561fd7bd --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/test_extending.py @@ -0,0 +1,118 @@ +from importlib.util import spec_from_file_location, module_from_spec +import os +import pathlib +import pytest +import shutil +import subprocess +import sys +import sysconfig +import textwrap +import warnings + +import numpy as np +from numpy.testing import IS_WASM + + +try: + import cffi +except ImportError: + cffi = None + +if sys.flags.optimize > 1: + # no docstrings present to inspect when PYTHONOPTIMIZE/Py_OptimizeFlag > 1 + # cffi cannot succeed + cffi = None + +try: + with warnings.catch_warnings(record=True) as w: + # numba issue gh-4733 + warnings.filterwarnings('always', '', DeprecationWarning) + import numba +except (ImportError, SystemError): + # Certain numpy/numba versions trigger a SystemError due to a numba bug + numba = None + +try: + import cython + from Cython.Compiler.Version import version as cython_version +except ImportError: + cython = None +else: + from numpy._utils import _pep440 + # Cython 0.29.30 is required for Python 3.11 and there are + # other fixes in the 0.29 series that are needed even for earlier + # Python versions. + # Note: keep in sync with the one in pyproject.toml + required_version = '0.29.35' + if _pep440.parse(cython_version) < _pep440.Version(required_version): + # too old or wrong cython, skip the test + cython = None + + +@pytest.mark.skipif( + sys.platform == "win32" and sys.maxsize < 2**32, + reason="Failing in 32-bit Windows wheel build job, skip for now" +) +@pytest.mark.skipif(IS_WASM, reason="Can't start subprocess") +@pytest.mark.skipif(cython is None, reason="requires cython") +@pytest.mark.slow +def test_cython(tmp_path): + import glob + # build the examples in a temporary directory + srcdir = os.path.join(os.path.dirname(__file__), '..') + shutil.copytree(srcdir, tmp_path / 'random') + build_dir = tmp_path / 'random' / '_examples' / 'cython' + target_dir = build_dir / "build" + os.makedirs(target_dir, exist_ok=True) + if sys.platform == "win32": + subprocess.check_call(["meson", "setup", + "--buildtype=release", + "--vsenv", str(build_dir)], + cwd=target_dir, + ) + else: + subprocess.check_call(["meson", "setup", str(build_dir)], + cwd=target_dir + ) + subprocess.check_call(["meson", "compile", "-vv"], cwd=target_dir) + + # gh-16162: make sure numpy's __init__.pxd was used for cython + # not really part of this test, but it is a convenient place to check + + g = glob.glob(str(target_dir / "*" / "extending.pyx.c")) + with open(g[0]) as fid: + txt_to_find = 'NumPy API declarations from "numpy/__init__' + for i, line in enumerate(fid): + if txt_to_find in line: + break + else: + assert False, ("Could not find '{}' in C file, " + "wrong pxd used".format(txt_to_find)) + # import without adding the directory to sys.path + suffix = sysconfig.get_config_var('EXT_SUFFIX') + + def load(modname): + so = (target_dir / modname).with_suffix(suffix) + spec = spec_from_file_location(modname, so) + mod = module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + # test that the module can be imported + load("extending") + load("extending_cpp") + # actually test the cython c-extension + extending_distributions = load("extending_distributions") + from numpy.random import PCG64 + values = extending_distributions.uniforms_ex(PCG64(0), 10, 'd') + assert values.shape == (10,) + assert values.dtype == np.float64 + +@pytest.mark.skipif(numba is None or cffi is None, + reason="requires numba and cffi") +def test_numba(): + from numpy.random._examples.numba import extending # noqa: F401 + +@pytest.mark.skipif(cffi is None, reason="requires cffi") +def test_cffi(): + from numpy.random._examples.cffi import extending # noqa: F401 diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/test_generator_mt19937.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/test_generator_mt19937.py new file mode 100644 index 0000000000000000000000000000000000000000..e744f5ba611b177b10034cada76f0dd28f63cf16 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/test_generator_mt19937.py @@ -0,0 +1,2746 @@ +import sys +import hashlib + +import pytest + +import numpy as np +from numpy.linalg import LinAlgError +from numpy.testing import ( + assert_, assert_raises, assert_equal, assert_allclose, + assert_warns, assert_no_warnings, assert_array_equal, + assert_array_almost_equal, suppress_warnings, IS_WASM) + +from numpy.random import Generator, MT19937, SeedSequence, RandomState + +random = Generator(MT19937()) + +JUMP_TEST_DATA = [ + { + "seed": 0, + "steps": 10, + "initial": {"key_sha256": "bb1636883c2707b51c5b7fc26c6927af4430f2e0785a8c7bc886337f919f9edf", "pos": 9}, + "jumped": {"key_sha256": "ff682ac12bb140f2d72fba8d3506cf4e46817a0db27aae1683867629031d8d55", "pos": 598}, + }, + { + "seed":384908324, + "steps":312, + "initial": {"key_sha256": "16b791a1e04886ccbbb4d448d6ff791267dc458ae599475d08d5cced29d11614", "pos": 311}, + "jumped": {"key_sha256": "a0110a2cf23b56be0feaed8f787a7fc84bef0cb5623003d75b26bdfa1c18002c", "pos": 276}, + }, + { + "seed": [839438204, 980239840, 859048019, 821], + "steps": 511, + "initial": {"key_sha256": "d306cf01314d51bd37892d874308200951a35265ede54d200f1e065004c3e9ea", "pos": 510}, + "jumped": {"key_sha256": "0e00ab449f01a5195a83b4aee0dfbc2ce8d46466a640b92e33977d2e42f777f8", "pos": 475}, + }, +] + + +@pytest.fixture(scope='module', params=[True, False]) +def endpoint(request): + return request.param + + +class TestSeed: + def test_scalar(self): + s = Generator(MT19937(0)) + assert_equal(s.integers(1000), 479) + s = Generator(MT19937(4294967295)) + assert_equal(s.integers(1000), 324) + + def test_array(self): + s = Generator(MT19937(range(10))) + assert_equal(s.integers(1000), 465) + s = Generator(MT19937(np.arange(10))) + assert_equal(s.integers(1000), 465) + s = Generator(MT19937([0])) + assert_equal(s.integers(1000), 479) + s = Generator(MT19937([4294967295])) + assert_equal(s.integers(1000), 324) + + def test_seedsequence(self): + s = MT19937(SeedSequence(0)) + assert_equal(s.random_raw(1), 2058676884) + + def test_invalid_scalar(self): + # seed must be an unsigned 32 bit integer + assert_raises(TypeError, MT19937, -0.5) + assert_raises(ValueError, MT19937, -1) + + def test_invalid_array(self): + # seed must be an unsigned integer + assert_raises(TypeError, MT19937, [-0.5]) + assert_raises(ValueError, MT19937, [-1]) + assert_raises(ValueError, MT19937, [1, -2, 4294967296]) + + def test_noninstantized_bitgen(self): + assert_raises(ValueError, Generator, MT19937) + + +class TestBinomial: + def test_n_zero(self): + # Tests the corner case of n == 0 for the binomial distribution. + # binomial(0, p) should be zero for any p in [0, 1]. + # This test addresses issue #3480. + zeros = np.zeros(2, dtype='int') + for p in [0, .5, 1]: + assert_(random.binomial(0, p) == 0) + assert_array_equal(random.binomial(zeros, p), zeros) + + def test_p_is_nan(self): + # Issue #4571. + assert_raises(ValueError, random.binomial, 1, np.nan) + + +class TestMultinomial: + def test_basic(self): + random.multinomial(100, [0.2, 0.8]) + + def test_zero_probability(self): + random.multinomial(100, [0.2, 0.8, 0.0, 0.0, 0.0]) + + def test_int_negative_interval(self): + assert_(-5 <= random.integers(-5, -1) < -1) + x = random.integers(-5, -1, 5) + assert_(np.all(-5 <= x)) + assert_(np.all(x < -1)) + + def test_size(self): + # gh-3173 + p = [0.5, 0.5] + assert_equal(random.multinomial(1, p, np.uint32(1)).shape, (1, 2)) + assert_equal(random.multinomial(1, p, np.uint32(1)).shape, (1, 2)) + assert_equal(random.multinomial(1, p, np.uint32(1)).shape, (1, 2)) + assert_equal(random.multinomial(1, p, [2, 2]).shape, (2, 2, 2)) + assert_equal(random.multinomial(1, p, (2, 2)).shape, (2, 2, 2)) + assert_equal(random.multinomial(1, p, np.array((2, 2))).shape, + (2, 2, 2)) + + assert_raises(TypeError, random.multinomial, 1, p, + float(1)) + + def test_invalid_prob(self): + assert_raises(ValueError, random.multinomial, 100, [1.1, 0.2]) + assert_raises(ValueError, random.multinomial, 100, [-.1, 0.9]) + + def test_invalid_n(self): + assert_raises(ValueError, random.multinomial, -1, [0.8, 0.2]) + assert_raises(ValueError, random.multinomial, [-1] * 10, [0.8, 0.2]) + + def test_p_non_contiguous(self): + p = np.arange(15.) + p /= np.sum(p[1::3]) + pvals = p[1::3] + random = Generator(MT19937(1432985819)) + non_contig = random.multinomial(100, pvals=pvals) + random = Generator(MT19937(1432985819)) + contig = random.multinomial(100, pvals=np.ascontiguousarray(pvals)) + assert_array_equal(non_contig, contig) + + def test_multinomial_pvals_float32(self): + x = np.array([9.9e-01, 9.9e-01, 1.0e-09, 1.0e-09, 1.0e-09, 1.0e-09, + 1.0e-09, 1.0e-09, 1.0e-09, 1.0e-09], dtype=np.float32) + pvals = x / x.sum() + random = Generator(MT19937(1432985819)) + match = r"[\w\s]*pvals array is cast to 64-bit floating" + with pytest.raises(ValueError, match=match): + random.multinomial(1, pvals) + + +class TestMultivariateHypergeometric: + + def setup_method(self): + self.seed = 8675309 + + def test_argument_validation(self): + # Error cases... + + # `colors` must be a 1-d sequence + assert_raises(ValueError, random.multivariate_hypergeometric, + 10, 4) + + # Negative nsample + assert_raises(ValueError, random.multivariate_hypergeometric, + [2, 3, 4], -1) + + # Negative color + assert_raises(ValueError, random.multivariate_hypergeometric, + [-1, 2, 3], 2) + + # nsample exceeds sum(colors) + assert_raises(ValueError, random.multivariate_hypergeometric, + [2, 3, 4], 10) + + # nsample exceeds sum(colors) (edge case of empty colors) + assert_raises(ValueError, random.multivariate_hypergeometric, + [], 1) + + # Validation errors associated with very large values in colors. + assert_raises(ValueError, random.multivariate_hypergeometric, + [999999999, 101], 5, 1, 'marginals') + + int64_info = np.iinfo(np.int64) + max_int64 = int64_info.max + max_int64_index = max_int64 // int64_info.dtype.itemsize + assert_raises(ValueError, random.multivariate_hypergeometric, + [max_int64_index - 100, 101], 5, 1, 'count') + + @pytest.mark.parametrize('method', ['count', 'marginals']) + def test_edge_cases(self, method): + # Set the seed, but in fact, all the results in this test are + # deterministic, so we don't really need this. + random = Generator(MT19937(self.seed)) + + x = random.multivariate_hypergeometric([0, 0, 0], 0, method=method) + assert_array_equal(x, [0, 0, 0]) + + x = random.multivariate_hypergeometric([], 0, method=method) + assert_array_equal(x, []) + + x = random.multivariate_hypergeometric([], 0, size=1, method=method) + assert_array_equal(x, np.empty((1, 0), dtype=np.int64)) + + x = random.multivariate_hypergeometric([1, 2, 3], 0, method=method) + assert_array_equal(x, [0, 0, 0]) + + x = random.multivariate_hypergeometric([9, 0, 0], 3, method=method) + assert_array_equal(x, [3, 0, 0]) + + colors = [1, 1, 0, 1, 1] + x = random.multivariate_hypergeometric(colors, sum(colors), + method=method) + assert_array_equal(x, colors) + + x = random.multivariate_hypergeometric([3, 4, 5], 12, size=3, + method=method) + assert_array_equal(x, [[3, 4, 5]]*3) + + # Cases for nsample: + # nsample < 10 + # 10 <= nsample < colors.sum()/2 + # colors.sum()/2 < nsample < colors.sum() - 10 + # colors.sum() - 10 < nsample < colors.sum() + @pytest.mark.parametrize('nsample', [8, 25, 45, 55]) + @pytest.mark.parametrize('method', ['count', 'marginals']) + @pytest.mark.parametrize('size', [5, (2, 3), 150000]) + def test_typical_cases(self, nsample, method, size): + random = Generator(MT19937(self.seed)) + + colors = np.array([10, 5, 20, 25]) + sample = random.multivariate_hypergeometric(colors, nsample, size, + method=method) + if isinstance(size, int): + expected_shape = (size,) + colors.shape + else: + expected_shape = size + colors.shape + assert_equal(sample.shape, expected_shape) + assert_((sample >= 0).all()) + assert_((sample <= colors).all()) + assert_array_equal(sample.sum(axis=-1), + np.full(size, fill_value=nsample, dtype=int)) + if isinstance(size, int) and size >= 100000: + # This sample is large enough to compare its mean to + # the expected values. + assert_allclose(sample.mean(axis=0), + nsample * colors / colors.sum(), + rtol=1e-3, atol=0.005) + + def test_repeatability1(self): + random = Generator(MT19937(self.seed)) + sample = random.multivariate_hypergeometric([3, 4, 5], 5, size=5, + method='count') + expected = np.array([[2, 1, 2], + [2, 1, 2], + [1, 1, 3], + [2, 0, 3], + [2, 1, 2]]) + assert_array_equal(sample, expected) + + def test_repeatability2(self): + random = Generator(MT19937(self.seed)) + sample = random.multivariate_hypergeometric([20, 30, 50], 50, + size=5, + method='marginals') + expected = np.array([[ 9, 17, 24], + [ 7, 13, 30], + [ 9, 15, 26], + [ 9, 17, 24], + [12, 14, 24]]) + assert_array_equal(sample, expected) + + def test_repeatability3(self): + random = Generator(MT19937(self.seed)) + sample = random.multivariate_hypergeometric([20, 30, 50], 12, + size=5, + method='marginals') + expected = np.array([[2, 3, 7], + [5, 3, 4], + [2, 5, 5], + [5, 3, 4], + [1, 5, 6]]) + assert_array_equal(sample, expected) + + +class TestSetState: + def setup_method(self): + self.seed = 1234567890 + self.rg = Generator(MT19937(self.seed)) + self.bit_generator = self.rg.bit_generator + self.state = self.bit_generator.state + self.legacy_state = (self.state['bit_generator'], + self.state['state']['key'], + self.state['state']['pos']) + + def test_gaussian_reset(self): + # Make sure the cached every-other-Gaussian is reset. + old = self.rg.standard_normal(size=3) + self.bit_generator.state = self.state + new = self.rg.standard_normal(size=3) + assert_(np.all(old == new)) + + def test_gaussian_reset_in_media_res(self): + # When the state is saved with a cached Gaussian, make sure the + # cached Gaussian is restored. + + self.rg.standard_normal() + state = self.bit_generator.state + old = self.rg.standard_normal(size=3) + self.bit_generator.state = state + new = self.rg.standard_normal(size=3) + assert_(np.all(old == new)) + + def test_negative_binomial(self): + # Ensure that the negative binomial results take floating point + # arguments without truncation. + self.rg.negative_binomial(0.5, 0.5) + + +class TestIntegers: + rfunc = random.integers + + # valid integer/boolean types + itype = [bool, np.int8, np.uint8, np.int16, np.uint16, + np.int32, np.uint32, np.int64, np.uint64] + + def test_unsupported_type(self, endpoint): + assert_raises(TypeError, self.rfunc, 1, endpoint=endpoint, dtype=float) + + def test_bounds_checking(self, endpoint): + for dt in self.itype: + lbnd = 0 if dt is bool else np.iinfo(dt).min + ubnd = 2 if dt is bool else np.iinfo(dt).max + 1 + ubnd = ubnd - 1 if endpoint else ubnd + assert_raises(ValueError, self.rfunc, lbnd - 1, ubnd, + endpoint=endpoint, dtype=dt) + assert_raises(ValueError, self.rfunc, lbnd, ubnd + 1, + endpoint=endpoint, dtype=dt) + assert_raises(ValueError, self.rfunc, ubnd, lbnd, + endpoint=endpoint, dtype=dt) + assert_raises(ValueError, self.rfunc, 1, 0, endpoint=endpoint, + dtype=dt) + + assert_raises(ValueError, self.rfunc, [lbnd - 1], ubnd, + endpoint=endpoint, dtype=dt) + assert_raises(ValueError, self.rfunc, [lbnd], [ubnd + 1], + endpoint=endpoint, dtype=dt) + assert_raises(ValueError, self.rfunc, [ubnd], [lbnd], + endpoint=endpoint, dtype=dt) + assert_raises(ValueError, self.rfunc, 1, [0], + endpoint=endpoint, dtype=dt) + assert_raises(ValueError, self.rfunc, [ubnd+1], [ubnd], + endpoint=endpoint, dtype=dt) + + def test_bounds_checking_array(self, endpoint): + for dt in self.itype: + lbnd = 0 if dt is bool else np.iinfo(dt).min + ubnd = 2 if dt is bool else np.iinfo(dt).max + (not endpoint) + + assert_raises(ValueError, self.rfunc, [lbnd - 1] * 2, [ubnd] * 2, + endpoint=endpoint, dtype=dt) + assert_raises(ValueError, self.rfunc, [lbnd] * 2, + [ubnd + 1] * 2, endpoint=endpoint, dtype=dt) + assert_raises(ValueError, self.rfunc, ubnd, [lbnd] * 2, + endpoint=endpoint, dtype=dt) + assert_raises(ValueError, self.rfunc, [1] * 2, 0, + endpoint=endpoint, dtype=dt) + + def test_rng_zero_and_extremes(self, endpoint): + for dt in self.itype: + lbnd = 0 if dt is bool else np.iinfo(dt).min + ubnd = 2 if dt is bool else np.iinfo(dt).max + 1 + ubnd = ubnd - 1 if endpoint else ubnd + is_open = not endpoint + + tgt = ubnd - 1 + assert_equal(self.rfunc(tgt, tgt + is_open, size=1000, + endpoint=endpoint, dtype=dt), tgt) + assert_equal(self.rfunc([tgt], tgt + is_open, size=1000, + endpoint=endpoint, dtype=dt), tgt) + + tgt = lbnd + assert_equal(self.rfunc(tgt, tgt + is_open, size=1000, + endpoint=endpoint, dtype=dt), tgt) + assert_equal(self.rfunc(tgt, [tgt + is_open], size=1000, + endpoint=endpoint, dtype=dt), tgt) + + tgt = (lbnd + ubnd) // 2 + assert_equal(self.rfunc(tgt, tgt + is_open, size=1000, + endpoint=endpoint, dtype=dt), tgt) + assert_equal(self.rfunc([tgt], [tgt + is_open], + size=1000, endpoint=endpoint, dtype=dt), + tgt) + + def test_rng_zero_and_extremes_array(self, endpoint): + size = 1000 + for dt in self.itype: + lbnd = 0 if dt is bool else np.iinfo(dt).min + ubnd = 2 if dt is bool else np.iinfo(dt).max + 1 + ubnd = ubnd - 1 if endpoint else ubnd + + tgt = ubnd - 1 + assert_equal(self.rfunc([tgt], [tgt + 1], + size=size, dtype=dt), tgt) + assert_equal(self.rfunc( + [tgt] * size, [tgt + 1] * size, dtype=dt), tgt) + assert_equal(self.rfunc( + [tgt] * size, [tgt + 1] * size, size=size, dtype=dt), tgt) + + tgt = lbnd + assert_equal(self.rfunc([tgt], [tgt + 1], + size=size, dtype=dt), tgt) + assert_equal(self.rfunc( + [tgt] * size, [tgt + 1] * size, dtype=dt), tgt) + assert_equal(self.rfunc( + [tgt] * size, [tgt + 1] * size, size=size, dtype=dt), tgt) + + tgt = (lbnd + ubnd) // 2 + assert_equal(self.rfunc([tgt], [tgt + 1], + size=size, dtype=dt), tgt) + assert_equal(self.rfunc( + [tgt] * size, [tgt + 1] * size, dtype=dt), tgt) + assert_equal(self.rfunc( + [tgt] * size, [tgt + 1] * size, size=size, dtype=dt), tgt) + + def test_full_range(self, endpoint): + # Test for ticket #1690 + + for dt in self.itype: + lbnd = 0 if dt is bool else np.iinfo(dt).min + ubnd = 2 if dt is bool else np.iinfo(dt).max + 1 + ubnd = ubnd - 1 if endpoint else ubnd + + try: + self.rfunc(lbnd, ubnd, endpoint=endpoint, dtype=dt) + except Exception as e: + raise AssertionError("No error should have been raised, " + "but one was with the following " + "message:\n\n%s" % str(e)) + + def test_full_range_array(self, endpoint): + # Test for ticket #1690 + + for dt in self.itype: + lbnd = 0 if dt is bool else np.iinfo(dt).min + ubnd = 2 if dt is bool else np.iinfo(dt).max + 1 + ubnd = ubnd - 1 if endpoint else ubnd + + try: + self.rfunc([lbnd] * 2, [ubnd], endpoint=endpoint, dtype=dt) + except Exception as e: + raise AssertionError("No error should have been raised, " + "but one was with the following " + "message:\n\n%s" % str(e)) + + def test_in_bounds_fuzz(self, endpoint): + # Don't use fixed seed + random = Generator(MT19937()) + + for dt in self.itype[1:]: + for ubnd in [4, 8, 16]: + vals = self.rfunc(2, ubnd - endpoint, size=2 ** 16, + endpoint=endpoint, dtype=dt) + assert_(vals.max() < ubnd) + assert_(vals.min() >= 2) + + vals = self.rfunc(0, 2 - endpoint, size=2 ** 16, endpoint=endpoint, + dtype=bool) + assert_(vals.max() < 2) + assert_(vals.min() >= 0) + + def test_scalar_array_equiv(self, endpoint): + for dt in self.itype: + lbnd = 0 if dt is bool else np.iinfo(dt).min + ubnd = 2 if dt is bool else np.iinfo(dt).max + 1 + ubnd = ubnd - 1 if endpoint else ubnd + + size = 1000 + random = Generator(MT19937(1234)) + scalar = random.integers(lbnd, ubnd, size=size, endpoint=endpoint, + dtype=dt) + + random = Generator(MT19937(1234)) + scalar_array = random.integers([lbnd], [ubnd], size=size, + endpoint=endpoint, dtype=dt) + + random = Generator(MT19937(1234)) + array = random.integers([lbnd] * size, [ubnd] * + size, size=size, endpoint=endpoint, dtype=dt) + assert_array_equal(scalar, scalar_array) + assert_array_equal(scalar, array) + + def test_repeatability(self, endpoint): + # We use a sha256 hash of generated sequences of 1000 samples + # in the range [0, 6) for all but bool, where the range + # is [0, 2). Hashes are for little endian numbers. + tgt = {'bool': '053594a9b82d656f967c54869bc6970aa0358cf94ad469c81478459c6a90eee3', + 'int16': '54de9072b6ee9ff7f20b58329556a46a447a8a29d67db51201bf88baa6e4e5d4', + 'int32': 'd3a0d5efb04542b25ac712e50d21f39ac30f312a5052e9bbb1ad3baa791ac84b', + 'int64': '14e224389ac4580bfbdccb5697d6190b496f91227cf67df60989de3d546389b1', + 'int8': '0e203226ff3fbbd1580f15da4621e5f7164d0d8d6b51696dd42d004ece2cbec1', + 'uint16': '54de9072b6ee9ff7f20b58329556a46a447a8a29d67db51201bf88baa6e4e5d4', + 'uint32': 'd3a0d5efb04542b25ac712e50d21f39ac30f312a5052e9bbb1ad3baa791ac84b', + 'uint64': '14e224389ac4580bfbdccb5697d6190b496f91227cf67df60989de3d546389b1', + 'uint8': '0e203226ff3fbbd1580f15da4621e5f7164d0d8d6b51696dd42d004ece2cbec1'} + + for dt in self.itype[1:]: + random = Generator(MT19937(1234)) + + # view as little endian for hash + if sys.byteorder == 'little': + val = random.integers(0, 6 - endpoint, size=1000, endpoint=endpoint, + dtype=dt) + else: + val = random.integers(0, 6 - endpoint, size=1000, endpoint=endpoint, + dtype=dt).byteswap() + + res = hashlib.sha256(val).hexdigest() + assert_(tgt[np.dtype(dt).name] == res) + + # bools do not depend on endianness + random = Generator(MT19937(1234)) + val = random.integers(0, 2 - endpoint, size=1000, endpoint=endpoint, + dtype=bool).view(np.int8) + res = hashlib.sha256(val).hexdigest() + assert_(tgt[np.dtype(bool).name] == res) + + def test_repeatability_broadcasting(self, endpoint): + for dt in self.itype: + lbnd = 0 if dt in (bool, np.bool_) else np.iinfo(dt).min + ubnd = 2 if dt in (bool, np.bool_) else np.iinfo(dt).max + 1 + ubnd = ubnd - 1 if endpoint else ubnd + + # view as little endian for hash + random = Generator(MT19937(1234)) + val = random.integers(lbnd, ubnd, size=1000, endpoint=endpoint, + dtype=dt) + + random = Generator(MT19937(1234)) + val_bc = random.integers([lbnd] * 1000, ubnd, endpoint=endpoint, + dtype=dt) + + assert_array_equal(val, val_bc) + + random = Generator(MT19937(1234)) + val_bc = random.integers([lbnd] * 1000, [ubnd] * 1000, + endpoint=endpoint, dtype=dt) + + assert_array_equal(val, val_bc) + + @pytest.mark.parametrize( + 'bound, expected', + [(2**32 - 1, np.array([517043486, 1364798665, 1733884389, 1353720612, + 3769704066, 1170797179, 4108474671])), + (2**32, np.array([517043487, 1364798666, 1733884390, 1353720613, + 3769704067, 1170797180, 4108474672])), + (2**32 + 1, np.array([517043487, 1733884390, 3769704068, 4108474673, + 1831631863, 1215661561, 3869512430]))] + ) + def test_repeatability_32bit_boundary(self, bound, expected): + for size in [None, len(expected)]: + random = Generator(MT19937(1234)) + x = random.integers(bound, size=size) + assert_equal(x, expected if size is not None else expected[0]) + + def test_repeatability_32bit_boundary_broadcasting(self): + desired = np.array([[[1622936284, 3620788691, 1659384060], + [1417365545, 760222891, 1909653332], + [3788118662, 660249498, 4092002593]], + [[3625610153, 2979601262, 3844162757], + [ 685800658, 120261497, 2694012896], + [1207779440, 1586594375, 3854335050]], + [[3004074748, 2310761796, 3012642217], + [2067714190, 2786677879, 1363865881], + [ 791663441, 1867303284, 2169727960]], + [[1939603804, 1250951100, 298950036], + [1040128489, 3791912209, 3317053765], + [3155528714, 61360675, 2305155588]], + [[ 817688762, 1335621943, 3288952434], + [1770890872, 1102951817, 1957607470], + [3099996017, 798043451, 48334215]]]) + for size in [None, (5, 3, 3)]: + random = Generator(MT19937(12345)) + x = random.integers([[-1], [0], [1]], + [2**32 - 1, 2**32, 2**32 + 1], + size=size) + assert_array_equal(x, desired if size is not None else desired[0]) + + def test_int64_uint64_broadcast_exceptions(self, endpoint): + configs = {np.uint64: ((0, 2**65), (-1, 2**62), (10, 9), (0, 0)), + np.int64: ((0, 2**64), (-(2**64), 2**62), (10, 9), (0, 0), + (-2**63-1, -2**63-1))} + for dtype in configs: + for config in configs[dtype]: + low, high = config + high = high - endpoint + low_a = np.array([[low]*10]) + high_a = np.array([high] * 10) + assert_raises(ValueError, random.integers, low, high, + endpoint=endpoint, dtype=dtype) + assert_raises(ValueError, random.integers, low_a, high, + endpoint=endpoint, dtype=dtype) + assert_raises(ValueError, random.integers, low, high_a, + endpoint=endpoint, dtype=dtype) + assert_raises(ValueError, random.integers, low_a, high_a, + endpoint=endpoint, dtype=dtype) + + low_o = np.array([[low]*10], dtype=object) + high_o = np.array([high] * 10, dtype=object) + assert_raises(ValueError, random.integers, low_o, high, + endpoint=endpoint, dtype=dtype) + assert_raises(ValueError, random.integers, low, high_o, + endpoint=endpoint, dtype=dtype) + assert_raises(ValueError, random.integers, low_o, high_o, + endpoint=endpoint, dtype=dtype) + + def test_int64_uint64_corner_case(self, endpoint): + # When stored in Numpy arrays, `lbnd` is casted + # as np.int64, and `ubnd` is casted as np.uint64. + # Checking whether `lbnd` >= `ubnd` used to be + # done solely via direct comparison, which is incorrect + # because when Numpy tries to compare both numbers, + # it casts both to np.float64 because there is + # no integer superset of np.int64 and np.uint64. However, + # `ubnd` is too large to be represented in np.float64, + # causing it be round down to np.iinfo(np.int64).max, + # leading to a ValueError because `lbnd` now equals + # the new `ubnd`. + + dt = np.int64 + tgt = np.iinfo(np.int64).max + lbnd = np.int64(np.iinfo(np.int64).max) + ubnd = np.uint64(np.iinfo(np.int64).max + 1 - endpoint) + + # None of these function calls should + # generate a ValueError now. + actual = random.integers(lbnd, ubnd, endpoint=endpoint, dtype=dt) + assert_equal(actual, tgt) + + def test_respect_dtype_singleton(self, endpoint): + # See gh-7203 + for dt in self.itype: + lbnd = 0 if dt is bool else np.iinfo(dt).min + ubnd = 2 if dt is bool else np.iinfo(dt).max + 1 + ubnd = ubnd - 1 if endpoint else ubnd + dt = np.bool_ if dt is bool else dt + + sample = self.rfunc(lbnd, ubnd, endpoint=endpoint, dtype=dt) + assert_equal(sample.dtype, dt) + + for dt in (bool, int): + lbnd = 0 if dt is bool else np.iinfo(dt).min + ubnd = 2 if dt is bool else np.iinfo(dt).max + 1 + ubnd = ubnd - 1 if endpoint else ubnd + + # gh-7284: Ensure that we get Python data types + sample = self.rfunc(lbnd, ubnd, endpoint=endpoint, dtype=dt) + assert not hasattr(sample, 'dtype') + assert_equal(type(sample), dt) + + def test_respect_dtype_array(self, endpoint): + # See gh-7203 + for dt in self.itype: + lbnd = 0 if dt is bool else np.iinfo(dt).min + ubnd = 2 if dt is bool else np.iinfo(dt).max + 1 + ubnd = ubnd - 1 if endpoint else ubnd + dt = np.bool_ if dt is bool else dt + + sample = self.rfunc([lbnd], [ubnd], endpoint=endpoint, dtype=dt) + assert_equal(sample.dtype, dt) + sample = self.rfunc([lbnd] * 2, [ubnd] * 2, endpoint=endpoint, + dtype=dt) + assert_equal(sample.dtype, dt) + + def test_zero_size(self, endpoint): + # See gh-7203 + for dt in self.itype: + sample = self.rfunc(0, 0, (3, 0, 4), endpoint=endpoint, dtype=dt) + assert sample.shape == (3, 0, 4) + assert sample.dtype == dt + assert self.rfunc(0, -10, 0, endpoint=endpoint, + dtype=dt).shape == (0,) + assert_equal(random.integers(0, 0, size=(3, 0, 4)).shape, + (3, 0, 4)) + assert_equal(random.integers(0, -10, size=0).shape, (0,)) + assert_equal(random.integers(10, 10, size=0).shape, (0,)) + + def test_error_byteorder(self): + other_byteord_dt = 'i4' + with pytest.raises(ValueError): + random.integers(0, 200, size=10, dtype=other_byteord_dt) + + # chi2max is the maximum acceptable chi-squared value. + @pytest.mark.slow + @pytest.mark.parametrize('sample_size,high,dtype,chi2max', + [(5000000, 5, np.int8, 125.0), # p-value ~4.6e-25 + (5000000, 7, np.uint8, 150.0), # p-value ~7.7e-30 + (10000000, 2500, np.int16, 3300.0), # p-value ~3.0e-25 + (50000000, 5000, np.uint16, 6500.0), # p-value ~3.5e-25 + ]) + def test_integers_small_dtype_chisquared(self, sample_size, high, + dtype, chi2max): + # Regression test for gh-14774. + samples = random.integers(high, size=sample_size, dtype=dtype) + + values, counts = np.unique(samples, return_counts=True) + expected = sample_size / high + chi2 = ((counts - expected)**2 / expected).sum() + assert chi2 < chi2max + + +class TestRandomDist: + # Make sure the random distribution returns the correct value for a + # given seed + + def setup_method(self): + self.seed = 1234567890 + + def test_integers(self): + random = Generator(MT19937(self.seed)) + actual = random.integers(-99, 99, size=(3, 2)) + desired = np.array([[-80, -56], [41, 37], [-83, -16]]) + assert_array_equal(actual, desired) + + def test_integers_masked(self): + # Test masked rejection sampling algorithm to generate array of + # uint32 in an interval. + random = Generator(MT19937(self.seed)) + actual = random.integers(0, 99, size=(3, 2), dtype=np.uint32) + desired = np.array([[9, 21], [70, 68], [8, 41]], dtype=np.uint32) + assert_array_equal(actual, desired) + + def test_integers_closed(self): + random = Generator(MT19937(self.seed)) + actual = random.integers(-99, 99, size=(3, 2), endpoint=True) + desired = np.array([[-80, -56], [ 41, 38], [-83, -15]]) + assert_array_equal(actual, desired) + + def test_integers_max_int(self): + # Tests whether integers with closed=True can generate the + # maximum allowed Python int that can be converted + # into a C long. Previous implementations of this + # method have thrown an OverflowError when attempting + # to generate this integer. + actual = random.integers(np.iinfo('l').max, np.iinfo('l').max, + endpoint=True) + + desired = np.iinfo('l').max + assert_equal(actual, desired) + + def test_random(self): + random = Generator(MT19937(self.seed)) + actual = random.random((3, 2)) + desired = np.array([[0.096999199829214, 0.707517457682192], + [0.084364834598269, 0.767731206553125], + [0.665069021359413, 0.715487190596693]]) + assert_array_almost_equal(actual, desired, decimal=15) + + random = Generator(MT19937(self.seed)) + actual = random.random() + assert_array_almost_equal(actual, desired[0, 0], decimal=15) + + def test_random_float(self): + random = Generator(MT19937(self.seed)) + actual = random.random((3, 2)) + desired = np.array([[0.0969992 , 0.70751746], + [0.08436483, 0.76773121], + [0.66506902, 0.71548719]]) + assert_array_almost_equal(actual, desired, decimal=7) + + def test_random_float_scalar(self): + random = Generator(MT19937(self.seed)) + actual = random.random(dtype=np.float32) + desired = 0.0969992 + assert_array_almost_equal(actual, desired, decimal=7) + + @pytest.mark.parametrize('dtype, uint_view_type', + [(np.float32, np.uint32), + (np.float64, np.uint64)]) + def test_random_distribution_of_lsb(self, dtype, uint_view_type): + random = Generator(MT19937(self.seed)) + sample = random.random(100000, dtype=dtype) + num_ones_in_lsb = np.count_nonzero(sample.view(uint_view_type) & 1) + # The probability of a 1 in the least significant bit is 0.25. + # With a sample size of 100000, the probability that num_ones_in_lsb + # is outside the following range is less than 5e-11. + assert 24100 < num_ones_in_lsb < 25900 + + def test_random_unsupported_type(self): + assert_raises(TypeError, random.random, dtype='int32') + + def test_choice_uniform_replace(self): + random = Generator(MT19937(self.seed)) + actual = random.choice(4, 4) + desired = np.array([0, 0, 2, 2], dtype=np.int64) + assert_array_equal(actual, desired) + + def test_choice_nonuniform_replace(self): + random = Generator(MT19937(self.seed)) + actual = random.choice(4, 4, p=[0.4, 0.4, 0.1, 0.1]) + desired = np.array([0, 1, 0, 1], dtype=np.int64) + assert_array_equal(actual, desired) + + def test_choice_uniform_noreplace(self): + random = Generator(MT19937(self.seed)) + actual = random.choice(4, 3, replace=False) + desired = np.array([2, 0, 3], dtype=np.int64) + assert_array_equal(actual, desired) + actual = random.choice(4, 4, replace=False, shuffle=False) + desired = np.arange(4, dtype=np.int64) + assert_array_equal(actual, desired) + + def test_choice_nonuniform_noreplace(self): + random = Generator(MT19937(self.seed)) + actual = random.choice(4, 3, replace=False, p=[0.1, 0.3, 0.5, 0.1]) + desired = np.array([0, 2, 3], dtype=np.int64) + assert_array_equal(actual, desired) + + def test_choice_noninteger(self): + random = Generator(MT19937(self.seed)) + actual = random.choice(['a', 'b', 'c', 'd'], 4) + desired = np.array(['a', 'a', 'c', 'c']) + assert_array_equal(actual, desired) + + def test_choice_multidimensional_default_axis(self): + random = Generator(MT19937(self.seed)) + actual = random.choice([[0, 1], [2, 3], [4, 5], [6, 7]], 3) + desired = np.array([[0, 1], [0, 1], [4, 5]]) + assert_array_equal(actual, desired) + + def test_choice_multidimensional_custom_axis(self): + random = Generator(MT19937(self.seed)) + actual = random.choice([[0, 1], [2, 3], [4, 5], [6, 7]], 1, axis=1) + desired = np.array([[0], [2], [4], [6]]) + assert_array_equal(actual, desired) + + def test_choice_exceptions(self): + sample = random.choice + assert_raises(ValueError, sample, -1, 3) + assert_raises(ValueError, sample, 3., 3) + assert_raises(ValueError, sample, [], 3) + assert_raises(ValueError, sample, [1, 2, 3, 4], 3, + p=[[0.25, 0.25], [0.25, 0.25]]) + assert_raises(ValueError, sample, [1, 2], 3, p=[0.4, 0.4, 0.2]) + assert_raises(ValueError, sample, [1, 2], 3, p=[1.1, -0.1]) + assert_raises(ValueError, sample, [1, 2], 3, p=[0.4, 0.4]) + assert_raises(ValueError, sample, [1, 2, 3], 4, replace=False) + # gh-13087 + assert_raises(ValueError, sample, [1, 2, 3], -2, replace=False) + assert_raises(ValueError, sample, [1, 2, 3], (-1,), replace=False) + assert_raises(ValueError, sample, [1, 2, 3], (-1, 1), replace=False) + assert_raises(ValueError, sample, [1, 2, 3], 2, + replace=False, p=[1, 0, 0]) + + def test_choice_return_shape(self): + p = [0.1, 0.9] + # Check scalar + assert_(np.isscalar(random.choice(2, replace=True))) + assert_(np.isscalar(random.choice(2, replace=False))) + assert_(np.isscalar(random.choice(2, replace=True, p=p))) + assert_(np.isscalar(random.choice(2, replace=False, p=p))) + assert_(np.isscalar(random.choice([1, 2], replace=True))) + assert_(random.choice([None], replace=True) is None) + a = np.array([1, 2]) + arr = np.empty(1, dtype=object) + arr[0] = a + assert_(random.choice(arr, replace=True) is a) + + # Check 0-d array + s = tuple() + assert_(not np.isscalar(random.choice(2, s, replace=True))) + assert_(not np.isscalar(random.choice(2, s, replace=False))) + assert_(not np.isscalar(random.choice(2, s, replace=True, p=p))) + assert_(not np.isscalar(random.choice(2, s, replace=False, p=p))) + assert_(not np.isscalar(random.choice([1, 2], s, replace=True))) + assert_(random.choice([None], s, replace=True).ndim == 0) + a = np.array([1, 2]) + arr = np.empty(1, dtype=object) + arr[0] = a + assert_(random.choice(arr, s, replace=True).item() is a) + + # Check multi dimensional array + s = (2, 3) + p = [0.1, 0.1, 0.1, 0.1, 0.4, 0.2] + assert_equal(random.choice(6, s, replace=True).shape, s) + assert_equal(random.choice(6, s, replace=False).shape, s) + assert_equal(random.choice(6, s, replace=True, p=p).shape, s) + assert_equal(random.choice(6, s, replace=False, p=p).shape, s) + assert_equal(random.choice(np.arange(6), s, replace=True).shape, s) + + # Check zero-size + assert_equal(random.integers(0, 0, size=(3, 0, 4)).shape, (3, 0, 4)) + assert_equal(random.integers(0, -10, size=0).shape, (0,)) + assert_equal(random.integers(10, 10, size=0).shape, (0,)) + assert_equal(random.choice(0, size=0).shape, (0,)) + assert_equal(random.choice([], size=(0,)).shape, (0,)) + assert_equal(random.choice(['a', 'b'], size=(3, 0, 4)).shape, + (3, 0, 4)) + assert_raises(ValueError, random.choice, [], 10) + + def test_choice_nan_probabilities(self): + a = np.array([42, 1, 2]) + p = [None, None, None] + assert_raises(ValueError, random.choice, a, p=p) + + def test_choice_p_non_contiguous(self): + p = np.ones(10) / 5 + p[1::2] = 3.0 + random = Generator(MT19937(self.seed)) + non_contig = random.choice(5, 3, p=p[::2]) + random = Generator(MT19937(self.seed)) + contig = random.choice(5, 3, p=np.ascontiguousarray(p[::2])) + assert_array_equal(non_contig, contig) + + def test_choice_return_type(self): + # gh 9867 + p = np.ones(4) / 4. + actual = random.choice(4, 2) + assert actual.dtype == np.int64 + actual = random.choice(4, 2, replace=False) + assert actual.dtype == np.int64 + actual = random.choice(4, 2, p=p) + assert actual.dtype == np.int64 + actual = random.choice(4, 2, p=p, replace=False) + assert actual.dtype == np.int64 + + def test_choice_large_sample(self): + choice_hash = '4266599d12bfcfb815213303432341c06b4349f5455890446578877bb322e222' + random = Generator(MT19937(self.seed)) + actual = random.choice(10000, 5000, replace=False) + if sys.byteorder != 'little': + actual = actual.byteswap() + res = hashlib.sha256(actual.view(np.int8)).hexdigest() + assert_(choice_hash == res) + + def test_bytes(self): + random = Generator(MT19937(self.seed)) + actual = random.bytes(10) + desired = b'\x86\xf0\xd4\x18\xe1\x81\t8%\xdd' + assert_equal(actual, desired) + + def test_shuffle(self): + # Test lists, arrays (of various dtypes), and multidimensional versions + # of both, c-contiguous or not: + for conv in [lambda x: np.array([]), + lambda x: x, + lambda x: np.asarray(x).astype(np.int8), + lambda x: np.asarray(x).astype(np.float32), + lambda x: np.asarray(x).astype(np.complex64), + lambda x: np.asarray(x).astype(object), + lambda x: [(i, i) for i in x], + lambda x: np.asarray([[i, i] for i in x]), + lambda x: np.vstack([x, x]).T, + # gh-11442 + lambda x: (np.asarray([(i, i) for i in x], + [("a", int), ("b", int)]) + .view(np.recarray)), + # gh-4270 + lambda x: np.asarray([(i, i) for i in x], + [("a", object, (1,)), + ("b", np.int32, (1,))])]: + random = Generator(MT19937(self.seed)) + alist = conv([1, 2, 3, 4, 5, 6, 7, 8, 9, 0]) + random.shuffle(alist) + actual = alist + desired = conv([4, 1, 9, 8, 0, 5, 3, 6, 2, 7]) + assert_array_equal(actual, desired) + + def test_shuffle_custom_axis(self): + random = Generator(MT19937(self.seed)) + actual = np.arange(16).reshape((4, 4)) + random.shuffle(actual, axis=1) + desired = np.array([[ 0, 3, 1, 2], + [ 4, 7, 5, 6], + [ 8, 11, 9, 10], + [12, 15, 13, 14]]) + assert_array_equal(actual, desired) + random = Generator(MT19937(self.seed)) + actual = np.arange(16).reshape((4, 4)) + random.shuffle(actual, axis=-1) + assert_array_equal(actual, desired) + + def test_shuffle_custom_axis_empty(self): + random = Generator(MT19937(self.seed)) + desired = np.array([]).reshape((0, 6)) + for axis in (0, 1): + actual = np.array([]).reshape((0, 6)) + random.shuffle(actual, axis=axis) + assert_array_equal(actual, desired) + + def test_shuffle_axis_nonsquare(self): + y1 = np.arange(20).reshape(2, 10) + y2 = y1.copy() + random = Generator(MT19937(self.seed)) + random.shuffle(y1, axis=1) + random = Generator(MT19937(self.seed)) + random.shuffle(y2.T) + assert_array_equal(y1, y2) + + def test_shuffle_masked(self): + # gh-3263 + a = np.ma.masked_values(np.reshape(range(20), (5, 4)) % 3 - 1, -1) + b = np.ma.masked_values(np.arange(20) % 3 - 1, -1) + a_orig = a.copy() + b_orig = b.copy() + for i in range(50): + random.shuffle(a) + assert_equal( + sorted(a.data[~a.mask]), sorted(a_orig.data[~a_orig.mask])) + random.shuffle(b) + assert_equal( + sorted(b.data[~b.mask]), sorted(b_orig.data[~b_orig.mask])) + + def test_shuffle_exceptions(self): + random = Generator(MT19937(self.seed)) + arr = np.arange(10) + assert_raises(np.AxisError, random.shuffle, arr, 1) + arr = np.arange(9).reshape((3, 3)) + assert_raises(np.AxisError, random.shuffle, arr, 3) + assert_raises(TypeError, random.shuffle, arr, slice(1, 2, None)) + arr = [[1, 2, 3], [4, 5, 6]] + assert_raises(NotImplementedError, random.shuffle, arr, 1) + + arr = np.array(3) + assert_raises(TypeError, random.shuffle, arr) + arr = np.ones((3, 2)) + assert_raises(np.AxisError, random.shuffle, arr, 2) + + def test_shuffle_not_writeable(self): + random = Generator(MT19937(self.seed)) + a = np.zeros(5) + a.flags.writeable = False + with pytest.raises(ValueError, match='read-only'): + random.shuffle(a) + + def test_permutation(self): + random = Generator(MT19937(self.seed)) + alist = [1, 2, 3, 4, 5, 6, 7, 8, 9, 0] + actual = random.permutation(alist) + desired = [4, 1, 9, 8, 0, 5, 3, 6, 2, 7] + assert_array_equal(actual, desired) + + random = Generator(MT19937(self.seed)) + arr_2d = np.atleast_2d([1, 2, 3, 4, 5, 6, 7, 8, 9, 0]).T + actual = random.permutation(arr_2d) + assert_array_equal(actual, np.atleast_2d(desired).T) + + bad_x_str = "abcd" + assert_raises(np.AxisError, random.permutation, bad_x_str) + + bad_x_float = 1.2 + assert_raises(np.AxisError, random.permutation, bad_x_float) + + random = Generator(MT19937(self.seed)) + integer_val = 10 + desired = [3, 0, 8, 7, 9, 4, 2, 5, 1, 6] + + actual = random.permutation(integer_val) + assert_array_equal(actual, desired) + + def test_permutation_custom_axis(self): + a = np.arange(16).reshape((4, 4)) + desired = np.array([[ 0, 3, 1, 2], + [ 4, 7, 5, 6], + [ 8, 11, 9, 10], + [12, 15, 13, 14]]) + random = Generator(MT19937(self.seed)) + actual = random.permutation(a, axis=1) + assert_array_equal(actual, desired) + random = Generator(MT19937(self.seed)) + actual = random.permutation(a, axis=-1) + assert_array_equal(actual, desired) + + def test_permutation_exceptions(self): + random = Generator(MT19937(self.seed)) + arr = np.arange(10) + assert_raises(np.AxisError, random.permutation, arr, 1) + arr = np.arange(9).reshape((3, 3)) + assert_raises(np.AxisError, random.permutation, arr, 3) + assert_raises(TypeError, random.permutation, arr, slice(1, 2, None)) + + @pytest.mark.parametrize("dtype", [int, object]) + @pytest.mark.parametrize("axis, expected", + [(None, np.array([[3, 7, 0, 9, 10, 11], + [8, 4, 2, 5, 1, 6]])), + (0, np.array([[6, 1, 2, 9, 10, 11], + [0, 7, 8, 3, 4, 5]])), + (1, np.array([[ 5, 3, 4, 0, 2, 1], + [11, 9, 10, 6, 8, 7]]))]) + def test_permuted(self, dtype, axis, expected): + random = Generator(MT19937(self.seed)) + x = np.arange(12).reshape(2, 6).astype(dtype) + random.permuted(x, axis=axis, out=x) + assert_array_equal(x, expected) + + random = Generator(MT19937(self.seed)) + x = np.arange(12).reshape(2, 6).astype(dtype) + y = random.permuted(x, axis=axis) + assert y.dtype == dtype + assert_array_equal(y, expected) + + def test_permuted_with_strides(self): + random = Generator(MT19937(self.seed)) + x0 = np.arange(22).reshape(2, 11) + x1 = x0.copy() + x = x0[:, ::3] + y = random.permuted(x, axis=1, out=x) + expected = np.array([[0, 9, 3, 6], + [14, 20, 11, 17]]) + assert_array_equal(y, expected) + x1[:, ::3] = expected + # Verify that the original x0 was modified in-place as expected. + assert_array_equal(x1, x0) + + def test_permuted_empty(self): + y = random.permuted([]) + assert_array_equal(y, []) + + @pytest.mark.parametrize('outshape', [(2, 3), 5]) + def test_permuted_out_with_wrong_shape(self, outshape): + a = np.array([1, 2, 3]) + out = np.zeros(outshape, dtype=a.dtype) + with pytest.raises(ValueError, match='same shape'): + random.permuted(a, out=out) + + def test_permuted_out_with_wrong_type(self): + out = np.zeros((3, 5), dtype=np.int32) + x = np.ones((3, 5)) + with pytest.raises(TypeError, match='Cannot cast'): + random.permuted(x, axis=1, out=out) + + def test_permuted_not_writeable(self): + x = np.zeros((2, 5)) + x.flags.writeable = False + with pytest.raises(ValueError, match='read-only'): + random.permuted(x, axis=1, out=x) + + def test_beta(self): + random = Generator(MT19937(self.seed)) + actual = random.beta(.1, .9, size=(3, 2)) + desired = np.array( + [[1.083029353267698e-10, 2.449965303168024e-11], + [2.397085162969853e-02, 3.590779671820755e-08], + [2.830254190078299e-04, 1.744709918330393e-01]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_binomial(self): + random = Generator(MT19937(self.seed)) + actual = random.binomial(100.123, .456, size=(3, 2)) + desired = np.array([[42, 41], + [42, 48], + [44, 50]]) + assert_array_equal(actual, desired) + + random = Generator(MT19937(self.seed)) + actual = random.binomial(100.123, .456) + desired = 42 + assert_array_equal(actual, desired) + + def test_chisquare(self): + random = Generator(MT19937(self.seed)) + actual = random.chisquare(50, size=(3, 2)) + desired = np.array([[32.9850547060149, 39.0219480493301], + [56.2006134779419, 57.3474165711485], + [55.4243733880198, 55.4209797925213]]) + assert_array_almost_equal(actual, desired, decimal=13) + + def test_dirichlet(self): + random = Generator(MT19937(self.seed)) + alpha = np.array([51.72840233779265162, 39.74494232180943953]) + actual = random.dirichlet(alpha, size=(3, 2)) + desired = np.array([[[0.5439892869558927, 0.45601071304410745], + [0.5588917345860708, 0.4411082654139292 ]], + [[0.5632074165063435, 0.43679258349365657], + [0.54862581112627, 0.45137418887373015]], + [[0.49961831357047226, 0.5003816864295278 ], + [0.52374806183482, 0.47625193816517997]]]) + assert_array_almost_equal(actual, desired, decimal=15) + bad_alpha = np.array([5.4e-01, -1.0e-16]) + assert_raises(ValueError, random.dirichlet, bad_alpha) + + random = Generator(MT19937(self.seed)) + alpha = np.array([51.72840233779265162, 39.74494232180943953]) + actual = random.dirichlet(alpha) + assert_array_almost_equal(actual, desired[0, 0], decimal=15) + + def test_dirichlet_size(self): + # gh-3173 + p = np.array([51.72840233779265162, 39.74494232180943953]) + assert_equal(random.dirichlet(p, np.uint32(1)).shape, (1, 2)) + assert_equal(random.dirichlet(p, np.uint32(1)).shape, (1, 2)) + assert_equal(random.dirichlet(p, np.uint32(1)).shape, (1, 2)) + assert_equal(random.dirichlet(p, [2, 2]).shape, (2, 2, 2)) + assert_equal(random.dirichlet(p, (2, 2)).shape, (2, 2, 2)) + assert_equal(random.dirichlet(p, np.array((2, 2))).shape, (2, 2, 2)) + + assert_raises(TypeError, random.dirichlet, p, float(1)) + + def test_dirichlet_bad_alpha(self): + # gh-2089 + alpha = np.array([5.4e-01, -1.0e-16]) + assert_raises(ValueError, random.dirichlet, alpha) + + # gh-15876 + assert_raises(ValueError, random.dirichlet, [[5, 1]]) + assert_raises(ValueError, random.dirichlet, [[5], [1]]) + assert_raises(ValueError, random.dirichlet, [[[5], [1]], [[1], [5]]]) + assert_raises(ValueError, random.dirichlet, np.array([[5, 1], [1, 5]])) + + def test_dirichlet_alpha_non_contiguous(self): + a = np.array([51.72840233779265162, -1.0, 39.74494232180943953]) + alpha = a[::2] + random = Generator(MT19937(self.seed)) + non_contig = random.dirichlet(alpha, size=(3, 2)) + random = Generator(MT19937(self.seed)) + contig = random.dirichlet(np.ascontiguousarray(alpha), + size=(3, 2)) + assert_array_almost_equal(non_contig, contig) + + def test_dirichlet_small_alpha(self): + eps = 1.0e-9 # 1.0e-10 -> runtime x 10; 1e-11 -> runtime x 200, etc. + alpha = eps * np.array([1., 1.0e-3]) + random = Generator(MT19937(self.seed)) + actual = random.dirichlet(alpha, size=(3, 2)) + expected = np.array([ + [[1., 0.], + [1., 0.]], + [[1., 0.], + [1., 0.]], + [[1., 0.], + [1., 0.]] + ]) + assert_array_almost_equal(actual, expected, decimal=15) + + @pytest.mark.slow + def test_dirichlet_moderately_small_alpha(self): + # Use alpha.max() < 0.1 to trigger stick breaking code path + alpha = np.array([0.02, 0.04, 0.03]) + exact_mean = alpha / alpha.sum() + random = Generator(MT19937(self.seed)) + sample = random.dirichlet(alpha, size=20000000) + sample_mean = sample.mean(axis=0) + assert_allclose(sample_mean, exact_mean, rtol=1e-3) + + # This set of parameters includes inputs with alpha.max() >= 0.1 and + # alpha.max() < 0.1 to exercise both generation methods within the + # dirichlet code. + @pytest.mark.parametrize( + 'alpha', + [[5, 9, 0, 8], + [0.5, 0, 0, 0], + [1, 5, 0, 0, 1.5, 0, 0, 0], + [0.01, 0.03, 0, 0.005], + [1e-5, 0, 0, 0], + [0.002, 0.015, 0, 0, 0.04, 0, 0, 0], + [0.0], + [0, 0, 0]], + ) + def test_dirichlet_multiple_zeros_in_alpha(self, alpha): + alpha = np.array(alpha) + y = random.dirichlet(alpha) + assert_equal(y[alpha == 0], 0.0) + + def test_exponential(self): + random = Generator(MT19937(self.seed)) + actual = random.exponential(1.1234, size=(3, 2)) + desired = np.array([[0.098845481066258, 1.560752510746964], + [0.075730916041636, 1.769098974710777], + [1.488602544592235, 2.49684815275751 ]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_exponential_0(self): + assert_equal(random.exponential(scale=0), 0) + assert_raises(ValueError, random.exponential, scale=-0.) + + def test_f(self): + random = Generator(MT19937(self.seed)) + actual = random.f(12, 77, size=(3, 2)) + desired = np.array([[0.461720027077085, 1.100441958872451], + [1.100337455217484, 0.91421736740018 ], + [0.500811891303113, 0.826802454552058]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_gamma(self): + random = Generator(MT19937(self.seed)) + actual = random.gamma(5, 3, size=(3, 2)) + desired = np.array([[ 5.03850858902096, 7.9228656732049 ], + [18.73983605132985, 19.57961681699238], + [18.17897755150825, 18.17653912505234]]) + assert_array_almost_equal(actual, desired, decimal=14) + + def test_gamma_0(self): + assert_equal(random.gamma(shape=0, scale=0), 0) + assert_raises(ValueError, random.gamma, shape=-0., scale=-0.) + + def test_geometric(self): + random = Generator(MT19937(self.seed)) + actual = random.geometric(.123456789, size=(3, 2)) + desired = np.array([[1, 11], + [1, 12], + [11, 17]]) + assert_array_equal(actual, desired) + + def test_geometric_exceptions(self): + assert_raises(ValueError, random.geometric, 1.1) + assert_raises(ValueError, random.geometric, [1.1] * 10) + assert_raises(ValueError, random.geometric, -0.1) + assert_raises(ValueError, random.geometric, [-0.1] * 10) + with np.errstate(invalid='ignore'): + assert_raises(ValueError, random.geometric, np.nan) + assert_raises(ValueError, random.geometric, [np.nan] * 10) + + def test_gumbel(self): + random = Generator(MT19937(self.seed)) + actual = random.gumbel(loc=.123456789, scale=2.0, size=(3, 2)) + desired = np.array([[ 4.688397515056245, -0.289514845417841], + [ 4.981176042584683, -0.633224272589149], + [-0.055915275687488, -0.333962478257953]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_gumbel_0(self): + assert_equal(random.gumbel(scale=0), 0) + assert_raises(ValueError, random.gumbel, scale=-0.) + + def test_hypergeometric(self): + random = Generator(MT19937(self.seed)) + actual = random.hypergeometric(10.1, 5.5, 14, size=(3, 2)) + desired = np.array([[ 9, 9], + [ 9, 9], + [10, 9]]) + assert_array_equal(actual, desired) + + # Test nbad = 0 + actual = random.hypergeometric(5, 0, 3, size=4) + desired = np.array([3, 3, 3, 3]) + assert_array_equal(actual, desired) + + actual = random.hypergeometric(15, 0, 12, size=4) + desired = np.array([12, 12, 12, 12]) + assert_array_equal(actual, desired) + + # Test ngood = 0 + actual = random.hypergeometric(0, 5, 3, size=4) + desired = np.array([0, 0, 0, 0]) + assert_array_equal(actual, desired) + + actual = random.hypergeometric(0, 15, 12, size=4) + desired = np.array([0, 0, 0, 0]) + assert_array_equal(actual, desired) + + def test_laplace(self): + random = Generator(MT19937(self.seed)) + actual = random.laplace(loc=.123456789, scale=2.0, size=(3, 2)) + desired = np.array([[-3.156353949272393, 1.195863024830054], + [-3.435458081645966, 1.656882398925444], + [ 0.924824032467446, 1.251116432209336]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_laplace_0(self): + assert_equal(random.laplace(scale=0), 0) + assert_raises(ValueError, random.laplace, scale=-0.) + + def test_logistic(self): + random = Generator(MT19937(self.seed)) + actual = random.logistic(loc=.123456789, scale=2.0, size=(3, 2)) + desired = np.array([[-4.338584631510999, 1.890171436749954], + [-4.64547787337966 , 2.514545562919217], + [ 1.495389489198666, 1.967827627577474]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_lognormal(self): + random = Generator(MT19937(self.seed)) + actual = random.lognormal(mean=.123456789, sigma=2.0, size=(3, 2)) + desired = np.array([[ 0.0268252166335, 13.9534486483053], + [ 0.1204014788936, 2.2422077497792], + [ 4.2484199496128, 12.0093343977523]]) + assert_array_almost_equal(actual, desired, decimal=13) + + def test_lognormal_0(self): + assert_equal(random.lognormal(sigma=0), 1) + assert_raises(ValueError, random.lognormal, sigma=-0.) + + def test_logseries(self): + random = Generator(MT19937(self.seed)) + actual = random.logseries(p=.923456789, size=(3, 2)) + desired = np.array([[14, 17], + [3, 18], + [5, 1]]) + assert_array_equal(actual, desired) + + def test_logseries_zero(self): + random = Generator(MT19937(self.seed)) + assert random.logseries(0) == 1 + + @pytest.mark.parametrize("value", [np.nextafter(0., -1), 1., np.nan, 5.]) + def test_logseries_exceptions(self, value): + random = Generator(MT19937(self.seed)) + with np.errstate(invalid="ignore"): + with pytest.raises(ValueError): + random.logseries(value) + with pytest.raises(ValueError): + # contiguous path: + random.logseries(np.array([value] * 10)) + with pytest.raises(ValueError): + # non-contiguous path: + random.logseries(np.array([value] * 10)[::2]) + + def test_multinomial(self): + random = Generator(MT19937(self.seed)) + actual = random.multinomial(20, [1 / 6.] * 6, size=(3, 2)) + desired = np.array([[[1, 5, 1, 6, 4, 3], + [4, 2, 6, 2, 4, 2]], + [[5, 3, 2, 6, 3, 1], + [4, 4, 0, 2, 3, 7]], + [[6, 3, 1, 5, 3, 2], + [5, 5, 3, 1, 2, 4]]]) + assert_array_equal(actual, desired) + + @pytest.mark.skipif(IS_WASM, reason="fp errors don't work in wasm") + @pytest.mark.parametrize("method", ["svd", "eigh", "cholesky"]) + def test_multivariate_normal(self, method): + random = Generator(MT19937(self.seed)) + mean = (.123456789, 10) + cov = [[1, 0], [0, 1]] + size = (3, 2) + actual = random.multivariate_normal(mean, cov, size, method=method) + desired = np.array([[[-1.747478062846581, 11.25613495182354 ], + [-0.9967333370066214, 10.342002097029821 ]], + [[ 0.7850019631242964, 11.181113712443013 ], + [ 0.8901349653255224, 8.873825399642492 ]], + [[ 0.7130260107430003, 9.551628690083056 ], + [ 0.7127098726541128, 11.991709234143173 ]]]) + + assert_array_almost_equal(actual, desired, decimal=15) + + # Check for default size, was raising deprecation warning + actual = random.multivariate_normal(mean, cov, method=method) + desired = np.array([0.233278563284287, 9.424140804347195]) + assert_array_almost_equal(actual, desired, decimal=15) + # Check that non symmetric covariance input raises exception when + # check_valid='raises' if using default svd method. + mean = [0, 0] + cov = [[1, 2], [1, 2]] + assert_raises(ValueError, random.multivariate_normal, mean, cov, + check_valid='raise') + + # Check that non positive-semidefinite covariance warns with + # RuntimeWarning + cov = [[1, 2], [2, 1]] + assert_warns(RuntimeWarning, random.multivariate_normal, mean, cov) + assert_warns(RuntimeWarning, random.multivariate_normal, mean, cov, + method='eigh') + assert_raises(LinAlgError, random.multivariate_normal, mean, cov, + method='cholesky') + + # and that it doesn't warn with RuntimeWarning check_valid='ignore' + assert_no_warnings(random.multivariate_normal, mean, cov, + check_valid='ignore') + + # and that it raises with RuntimeWarning check_valid='raises' + assert_raises(ValueError, random.multivariate_normal, mean, cov, + check_valid='raise') + assert_raises(ValueError, random.multivariate_normal, mean, cov, + check_valid='raise', method='eigh') + + # check degenerate samples from singular covariance matrix + cov = [[1, 1], [1, 1]] + if method in ('svd', 'eigh'): + samples = random.multivariate_normal(mean, cov, size=(3, 2), + method=method) + assert_array_almost_equal(samples[..., 0], samples[..., 1], + decimal=6) + else: + assert_raises(LinAlgError, random.multivariate_normal, mean, cov, + method='cholesky') + + cov = np.array([[1, 0.1], [0.1, 1]], dtype=np.float32) + with suppress_warnings() as sup: + random.multivariate_normal(mean, cov, method=method) + w = sup.record(RuntimeWarning) + assert len(w) == 0 + + mu = np.zeros(2) + cov = np.eye(2) + assert_raises(ValueError, random.multivariate_normal, mean, cov, + check_valid='other') + assert_raises(ValueError, random.multivariate_normal, + np.zeros((2, 1, 1)), cov) + assert_raises(ValueError, random.multivariate_normal, + mu, np.empty((3, 2))) + assert_raises(ValueError, random.multivariate_normal, + mu, np.eye(3)) + + @pytest.mark.parametrize('mean, cov', [([0], [[1+1j]]), ([0j], [[1]])]) + def test_multivariate_normal_disallow_complex(self, mean, cov): + random = Generator(MT19937(self.seed)) + with pytest.raises(TypeError, match="must not be complex"): + random.multivariate_normal(mean, cov) + + @pytest.mark.parametrize("method", ["svd", "eigh", "cholesky"]) + def test_multivariate_normal_basic_stats(self, method): + random = Generator(MT19937(self.seed)) + n_s = 1000 + mean = np.array([1, 2]) + cov = np.array([[2, 1], [1, 2]]) + s = random.multivariate_normal(mean, cov, size=(n_s,), method=method) + s_center = s - mean + cov_emp = (s_center.T @ s_center) / (n_s - 1) + # these are pretty loose and are only designed to detect major errors + assert np.all(np.abs(s_center.mean(-2)) < 0.1) + assert np.all(np.abs(cov_emp - cov) < 0.2) + + def test_negative_binomial(self): + random = Generator(MT19937(self.seed)) + actual = random.negative_binomial(n=100, p=.12345, size=(3, 2)) + desired = np.array([[543, 727], + [775, 760], + [600, 674]]) + assert_array_equal(actual, desired) + + def test_negative_binomial_exceptions(self): + with np.errstate(invalid='ignore'): + assert_raises(ValueError, random.negative_binomial, 100, np.nan) + assert_raises(ValueError, random.negative_binomial, 100, + [np.nan] * 10) + + def test_negative_binomial_p0_exception(self): + # Verify that p=0 raises an exception. + with assert_raises(ValueError): + x = random.negative_binomial(1, 0) + + def test_negative_binomial_invalid_p_n_combination(self): + # Verify that values of p and n that would result in an overflow + # or infinite loop raise an exception. + with np.errstate(invalid='ignore'): + assert_raises(ValueError, random.negative_binomial, 2**62, 0.1) + assert_raises(ValueError, random.negative_binomial, [2**62], [0.1]) + + def test_noncentral_chisquare(self): + random = Generator(MT19937(self.seed)) + actual = random.noncentral_chisquare(df=5, nonc=5, size=(3, 2)) + desired = np.array([[ 1.70561552362133, 15.97378184942111], + [13.71483425173724, 20.17859633310629], + [11.3615477156643 , 3.67891108738029]]) + assert_array_almost_equal(actual, desired, decimal=14) + + actual = random.noncentral_chisquare(df=.5, nonc=.2, size=(3, 2)) + desired = np.array([[9.41427665607629e-04, 1.70473157518850e-04], + [1.14554372041263e+00, 1.38187755933435e-03], + [1.90659181905387e+00, 1.21772577941822e+00]]) + assert_array_almost_equal(actual, desired, decimal=14) + + random = Generator(MT19937(self.seed)) + actual = random.noncentral_chisquare(df=5, nonc=0, size=(3, 2)) + desired = np.array([[0.82947954590419, 1.80139670767078], + [6.58720057417794, 7.00491463609814], + [6.31101879073157, 6.30982307753005]]) + assert_array_almost_equal(actual, desired, decimal=14) + + def test_noncentral_f(self): + random = Generator(MT19937(self.seed)) + actual = random.noncentral_f(dfnum=5, dfden=2, nonc=1, + size=(3, 2)) + desired = np.array([[0.060310671139 , 0.23866058175939], + [0.86860246709073, 0.2668510459738 ], + [0.23375780078364, 1.88922102885943]]) + assert_array_almost_equal(actual, desired, decimal=14) + + def test_noncentral_f_nan(self): + random = Generator(MT19937(self.seed)) + actual = random.noncentral_f(dfnum=5, dfden=2, nonc=np.nan) + assert np.isnan(actual) + + def test_normal(self): + random = Generator(MT19937(self.seed)) + actual = random.normal(loc=.123456789, scale=2.0, size=(3, 2)) + desired = np.array([[-3.618412914693162, 2.635726692647081], + [-2.116923463013243, 0.807460983059643], + [ 1.446547137248593, 2.485684213886024]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_normal_0(self): + assert_equal(random.normal(scale=0), 0) + assert_raises(ValueError, random.normal, scale=-0.) + + def test_pareto(self): + random = Generator(MT19937(self.seed)) + actual = random.pareto(a=.123456789, size=(3, 2)) + desired = np.array([[1.0394926776069018e+00, 7.7142534343505773e+04], + [7.2640150889064703e-01, 3.4650454783825594e+05], + [4.5852344481994740e+04, 6.5851383009539105e+07]]) + # For some reason on 32-bit x86 Ubuntu 12.10 the [1, 0] entry in this + # matrix differs by 24 nulps. Discussion: + # https://mail.python.org/pipermail/numpy-discussion/2012-September/063801.html + # Consensus is that this is probably some gcc quirk that affects + # rounding but not in any important way, so we just use a looser + # tolerance on this test: + np.testing.assert_array_almost_equal_nulp(actual, desired, nulp=30) + + def test_poisson(self): + random = Generator(MT19937(self.seed)) + actual = random.poisson(lam=.123456789, size=(3, 2)) + desired = np.array([[0, 0], + [0, 0], + [0, 0]]) + assert_array_equal(actual, desired) + + def test_poisson_exceptions(self): + lambig = np.iinfo('int64').max + lamneg = -1 + assert_raises(ValueError, random.poisson, lamneg) + assert_raises(ValueError, random.poisson, [lamneg] * 10) + assert_raises(ValueError, random.poisson, lambig) + assert_raises(ValueError, random.poisson, [lambig] * 10) + with np.errstate(invalid='ignore'): + assert_raises(ValueError, random.poisson, np.nan) + assert_raises(ValueError, random.poisson, [np.nan] * 10) + + def test_power(self): + random = Generator(MT19937(self.seed)) + actual = random.power(a=.123456789, size=(3, 2)) + desired = np.array([[1.977857368842754e-09, 9.806792196620341e-02], + [2.482442984543471e-10, 1.527108843266079e-01], + [8.188283434244285e-02, 3.950547209346948e-01]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_rayleigh(self): + random = Generator(MT19937(self.seed)) + actual = random.rayleigh(scale=10, size=(3, 2)) + desired = np.array([[4.19494429102666, 16.66920198906598], + [3.67184544902662, 17.74695521962917], + [16.27935397855501, 21.08355560691792]]) + assert_array_almost_equal(actual, desired, decimal=14) + + def test_rayleigh_0(self): + assert_equal(random.rayleigh(scale=0), 0) + assert_raises(ValueError, random.rayleigh, scale=-0.) + + def test_standard_cauchy(self): + random = Generator(MT19937(self.seed)) + actual = random.standard_cauchy(size=(3, 2)) + desired = np.array([[-1.489437778266206, -3.275389641569784], + [ 0.560102864910406, -0.680780916282552], + [-1.314912905226277, 0.295852965660225]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_standard_exponential(self): + random = Generator(MT19937(self.seed)) + actual = random.standard_exponential(size=(3, 2), method='inv') + desired = np.array([[0.102031839440643, 1.229350298474972], + [0.088137284693098, 1.459859985522667], + [1.093830802293668, 1.256977002164613]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_standard_expoential_type_error(self): + assert_raises(TypeError, random.standard_exponential, dtype=np.int32) + + def test_standard_gamma(self): + random = Generator(MT19937(self.seed)) + actual = random.standard_gamma(shape=3, size=(3, 2)) + desired = np.array([[0.62970724056362, 1.22379851271008], + [3.899412530884 , 4.12479964250139], + [3.74994102464584, 3.74929307690815]]) + assert_array_almost_equal(actual, desired, decimal=14) + + def test_standard_gammma_scalar_float(self): + random = Generator(MT19937(self.seed)) + actual = random.standard_gamma(3, dtype=np.float32) + desired = 2.9242148399353027 + assert_array_almost_equal(actual, desired, decimal=6) + + def test_standard_gamma_float(self): + random = Generator(MT19937(self.seed)) + actual = random.standard_gamma(shape=3, size=(3, 2)) + desired = np.array([[0.62971, 1.2238 ], + [3.89941, 4.1248 ], + [3.74994, 3.74929]]) + assert_array_almost_equal(actual, desired, decimal=5) + + def test_standard_gammma_float_out(self): + actual = np.zeros((3, 2), dtype=np.float32) + random = Generator(MT19937(self.seed)) + random.standard_gamma(10.0, out=actual, dtype=np.float32) + desired = np.array([[10.14987, 7.87012], + [ 9.46284, 12.56832], + [13.82495, 7.81533]], dtype=np.float32) + assert_array_almost_equal(actual, desired, decimal=5) + + random = Generator(MT19937(self.seed)) + random.standard_gamma(10.0, out=actual, size=(3, 2), dtype=np.float32) + assert_array_almost_equal(actual, desired, decimal=5) + + def test_standard_gamma_unknown_type(self): + assert_raises(TypeError, random.standard_gamma, 1., + dtype='int32') + + def test_out_size_mismatch(self): + out = np.zeros(10) + assert_raises(ValueError, random.standard_gamma, 10.0, size=20, + out=out) + assert_raises(ValueError, random.standard_gamma, 10.0, size=(10, 1), + out=out) + + def test_standard_gamma_0(self): + assert_equal(random.standard_gamma(shape=0), 0) + assert_raises(ValueError, random.standard_gamma, shape=-0.) + + def test_standard_normal(self): + random = Generator(MT19937(self.seed)) + actual = random.standard_normal(size=(3, 2)) + desired = np.array([[-1.870934851846581, 1.25613495182354 ], + [-1.120190126006621, 0.342002097029821], + [ 0.661545174124296, 1.181113712443012]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_standard_normal_unsupported_type(self): + assert_raises(TypeError, random.standard_normal, dtype=np.int32) + + def test_standard_t(self): + random = Generator(MT19937(self.seed)) + actual = random.standard_t(df=10, size=(3, 2)) + desired = np.array([[-1.484666193042647, 0.30597891831161 ], + [ 1.056684299648085, -0.407312602088507], + [ 0.130704414281157, -2.038053410490321]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_triangular(self): + random = Generator(MT19937(self.seed)) + actual = random.triangular(left=5.12, mode=10.23, right=20.34, + size=(3, 2)) + desired = np.array([[ 7.86664070590917, 13.6313848513185 ], + [ 7.68152445215983, 14.36169131136546], + [13.16105603911429, 13.72341621856971]]) + assert_array_almost_equal(actual, desired, decimal=14) + + def test_uniform(self): + random = Generator(MT19937(self.seed)) + actual = random.uniform(low=1.23, high=10.54, size=(3, 2)) + desired = np.array([[2.13306255040998 , 7.816987531021207], + [2.015436610109887, 8.377577533009589], + [7.421792588856135, 7.891185744455209]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_uniform_range_bounds(self): + fmin = np.finfo('float').min + fmax = np.finfo('float').max + + func = random.uniform + assert_raises(OverflowError, func, -np.inf, 0) + assert_raises(OverflowError, func, 0, np.inf) + assert_raises(OverflowError, func, fmin, fmax) + assert_raises(OverflowError, func, [-np.inf], [0]) + assert_raises(OverflowError, func, [0], [np.inf]) + + # (fmax / 1e17) - fmin is within range, so this should not throw + # account for i386 extended precision DBL_MAX / 1e17 + DBL_MAX > + # DBL_MAX by increasing fmin a bit + random.uniform(low=np.nextafter(fmin, 1), high=fmax / 1e17) + + def test_uniform_zero_range(self): + func = random.uniform + result = func(1.5, 1.5) + assert_allclose(result, 1.5) + result = func([0.0, np.pi], [0.0, np.pi]) + assert_allclose(result, [0.0, np.pi]) + result = func([[2145.12], [2145.12]], [2145.12, 2145.12]) + assert_allclose(result, 2145.12 + np.zeros((2, 2))) + + def test_uniform_neg_range(self): + func = random.uniform + assert_raises(ValueError, func, 2, 1) + assert_raises(ValueError, func, [1, 2], [1, 1]) + assert_raises(ValueError, func, [[0, 1],[2, 3]], 2) + + def test_scalar_exception_propagation(self): + # Tests that exceptions are correctly propagated in distributions + # when called with objects that throw exceptions when converted to + # scalars. + # + # Regression test for gh: 8865 + + class ThrowingFloat(np.ndarray): + def __float__(self): + raise TypeError + + throwing_float = np.array(1.0).view(ThrowingFloat) + assert_raises(TypeError, random.uniform, throwing_float, + throwing_float) + + class ThrowingInteger(np.ndarray): + def __int__(self): + raise TypeError + + throwing_int = np.array(1).view(ThrowingInteger) + assert_raises(TypeError, random.hypergeometric, throwing_int, 1, 1) + + def test_vonmises(self): + random = Generator(MT19937(self.seed)) + actual = random.vonmises(mu=1.23, kappa=1.54, size=(3, 2)) + desired = np.array([[ 1.107972248690106, 2.841536476232361], + [ 1.832602376042457, 1.945511926976032], + [-0.260147475776542, 2.058047492231698]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_vonmises_small(self): + # check infinite loop, gh-4720 + random = Generator(MT19937(self.seed)) + r = random.vonmises(mu=0., kappa=1.1e-8, size=10**6) + assert_(np.isfinite(r).all()) + + def test_vonmises_nan(self): + random = Generator(MT19937(self.seed)) + r = random.vonmises(mu=0., kappa=np.nan) + assert_(np.isnan(r)) + + @pytest.mark.parametrize("kappa", [1e4, 1e15]) + def test_vonmises_large_kappa(self, kappa): + random = Generator(MT19937(self.seed)) + rs = RandomState(random.bit_generator) + state = random.bit_generator.state + + random_state_vals = rs.vonmises(0, kappa, size=10) + random.bit_generator.state = state + gen_vals = random.vonmises(0, kappa, size=10) + if kappa < 1e6: + assert_allclose(random_state_vals, gen_vals) + else: + assert np.all(random_state_vals != gen_vals) + + @pytest.mark.parametrize("mu", [-7., -np.pi, -3.1, np.pi, 3.2]) + @pytest.mark.parametrize("kappa", [1e-9, 1e-6, 1, 1e3, 1e15]) + def test_vonmises_large_kappa_range(self, mu, kappa): + random = Generator(MT19937(self.seed)) + r = random.vonmises(mu, kappa, 50) + assert_(np.all(r > -np.pi) and np.all(r <= np.pi)) + + def test_wald(self): + random = Generator(MT19937(self.seed)) + actual = random.wald(mean=1.23, scale=1.54, size=(3, 2)) + desired = np.array([[0.26871721804551, 3.2233942732115 ], + [2.20328374987066, 2.40958405189353], + [2.07093587449261, 0.73073890064369]]) + assert_array_almost_equal(actual, desired, decimal=14) + + def test_weibull(self): + random = Generator(MT19937(self.seed)) + actual = random.weibull(a=1.23, size=(3, 2)) + desired = np.array([[0.138613914769468, 1.306463419753191], + [0.111623365934763, 1.446570494646721], + [1.257145775276011, 1.914247725027957]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_weibull_0(self): + random = Generator(MT19937(self.seed)) + assert_equal(random.weibull(a=0, size=12), np.zeros(12)) + assert_raises(ValueError, random.weibull, a=-0.) + + def test_zipf(self): + random = Generator(MT19937(self.seed)) + actual = random.zipf(a=1.23, size=(3, 2)) + desired = np.array([[ 1, 1], + [ 10, 867], + [354, 2]]) + assert_array_equal(actual, desired) + + +class TestBroadcast: + # tests that functions that broadcast behave + # correctly when presented with non-scalar arguments + def setup_method(self): + self.seed = 123456789 + + def test_uniform(self): + random = Generator(MT19937(self.seed)) + low = [0] + high = [1] + uniform = random.uniform + desired = np.array([0.16693771389729, 0.19635129550675, 0.75563050964095]) + + random = Generator(MT19937(self.seed)) + actual = random.uniform(low * 3, high) + assert_array_almost_equal(actual, desired, decimal=14) + + random = Generator(MT19937(self.seed)) + actual = random.uniform(low, high * 3) + assert_array_almost_equal(actual, desired, decimal=14) + + def test_normal(self): + loc = [0] + scale = [1] + bad_scale = [-1] + random = Generator(MT19937(self.seed)) + desired = np.array([-0.38736406738527, 0.79594375042255, 0.0197076236097]) + + random = Generator(MT19937(self.seed)) + actual = random.normal(loc * 3, scale) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, random.normal, loc * 3, bad_scale) + + random = Generator(MT19937(self.seed)) + normal = random.normal + actual = normal(loc, scale * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, normal, loc, bad_scale * 3) + + def test_beta(self): + a = [1] + b = [2] + bad_a = [-1] + bad_b = [-2] + desired = np.array([0.18719338682602, 0.73234824491364, 0.17928615186455]) + + random = Generator(MT19937(self.seed)) + beta = random.beta + actual = beta(a * 3, b) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, beta, bad_a * 3, b) + assert_raises(ValueError, beta, a * 3, bad_b) + + random = Generator(MT19937(self.seed)) + actual = random.beta(a, b * 3) + assert_array_almost_equal(actual, desired, decimal=14) + + def test_exponential(self): + scale = [1] + bad_scale = [-1] + desired = np.array([0.67245993212806, 0.21380495318094, 0.7177848928629]) + + random = Generator(MT19937(self.seed)) + actual = random.exponential(scale * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, random.exponential, bad_scale * 3) + + def test_standard_gamma(self): + shape = [1] + bad_shape = [-1] + desired = np.array([0.67245993212806, 0.21380495318094, 0.7177848928629]) + + random = Generator(MT19937(self.seed)) + std_gamma = random.standard_gamma + actual = std_gamma(shape * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, std_gamma, bad_shape * 3) + + def test_gamma(self): + shape = [1] + scale = [2] + bad_shape = [-1] + bad_scale = [-2] + desired = np.array([1.34491986425611, 0.42760990636187, 1.4355697857258]) + + random = Generator(MT19937(self.seed)) + gamma = random.gamma + actual = gamma(shape * 3, scale) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, gamma, bad_shape * 3, scale) + assert_raises(ValueError, gamma, shape * 3, bad_scale) + + random = Generator(MT19937(self.seed)) + gamma = random.gamma + actual = gamma(shape, scale * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, gamma, bad_shape, scale * 3) + assert_raises(ValueError, gamma, shape, bad_scale * 3) + + def test_f(self): + dfnum = [1] + dfden = [2] + bad_dfnum = [-1] + bad_dfden = [-2] + desired = np.array([0.07765056244107, 7.72951397913186, 0.05786093891763]) + + random = Generator(MT19937(self.seed)) + f = random.f + actual = f(dfnum * 3, dfden) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, f, bad_dfnum * 3, dfden) + assert_raises(ValueError, f, dfnum * 3, bad_dfden) + + random = Generator(MT19937(self.seed)) + f = random.f + actual = f(dfnum, dfden * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, f, bad_dfnum, dfden * 3) + assert_raises(ValueError, f, dfnum, bad_dfden * 3) + + def test_noncentral_f(self): + dfnum = [2] + dfden = [3] + nonc = [4] + bad_dfnum = [0] + bad_dfden = [-1] + bad_nonc = [-2] + desired = np.array([2.02434240411421, 12.91838601070124, 1.24395160354629]) + + random = Generator(MT19937(self.seed)) + nonc_f = random.noncentral_f + actual = nonc_f(dfnum * 3, dfden, nonc) + assert_array_almost_equal(actual, desired, decimal=14) + assert np.all(np.isnan(nonc_f(dfnum, dfden, [np.nan] * 3))) + + assert_raises(ValueError, nonc_f, bad_dfnum * 3, dfden, nonc) + assert_raises(ValueError, nonc_f, dfnum * 3, bad_dfden, nonc) + assert_raises(ValueError, nonc_f, dfnum * 3, dfden, bad_nonc) + + random = Generator(MT19937(self.seed)) + nonc_f = random.noncentral_f + actual = nonc_f(dfnum, dfden * 3, nonc) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, nonc_f, bad_dfnum, dfden * 3, nonc) + assert_raises(ValueError, nonc_f, dfnum, bad_dfden * 3, nonc) + assert_raises(ValueError, nonc_f, dfnum, dfden * 3, bad_nonc) + + random = Generator(MT19937(self.seed)) + nonc_f = random.noncentral_f + actual = nonc_f(dfnum, dfden, nonc * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, nonc_f, bad_dfnum, dfden, nonc * 3) + assert_raises(ValueError, nonc_f, dfnum, bad_dfden, nonc * 3) + assert_raises(ValueError, nonc_f, dfnum, dfden, bad_nonc * 3) + + def test_noncentral_f_small_df(self): + random = Generator(MT19937(self.seed)) + desired = np.array([0.04714867120827, 0.1239390327694]) + actual = random.noncentral_f(0.9, 0.9, 2, size=2) + assert_array_almost_equal(actual, desired, decimal=14) + + def test_chisquare(self): + df = [1] + bad_df = [-1] + desired = np.array([0.05573640064251, 1.47220224353539, 2.9469379318589]) + + random = Generator(MT19937(self.seed)) + actual = random.chisquare(df * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, random.chisquare, bad_df * 3) + + def test_noncentral_chisquare(self): + df = [1] + nonc = [2] + bad_df = [-1] + bad_nonc = [-2] + desired = np.array([0.07710766249436, 5.27829115110304, 0.630732147399]) + + random = Generator(MT19937(self.seed)) + nonc_chi = random.noncentral_chisquare + actual = nonc_chi(df * 3, nonc) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, nonc_chi, bad_df * 3, nonc) + assert_raises(ValueError, nonc_chi, df * 3, bad_nonc) + + random = Generator(MT19937(self.seed)) + nonc_chi = random.noncentral_chisquare + actual = nonc_chi(df, nonc * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, nonc_chi, bad_df, nonc * 3) + assert_raises(ValueError, nonc_chi, df, bad_nonc * 3) + + def test_standard_t(self): + df = [1] + bad_df = [-1] + desired = np.array([-1.39498829447098, -1.23058658835223, 0.17207021065983]) + + random = Generator(MT19937(self.seed)) + actual = random.standard_t(df * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, random.standard_t, bad_df * 3) + + def test_vonmises(self): + mu = [2] + kappa = [1] + bad_kappa = [-1] + desired = np.array([2.25935584988528, 2.23326261461399, -2.84152146503326]) + + random = Generator(MT19937(self.seed)) + actual = random.vonmises(mu * 3, kappa) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, random.vonmises, mu * 3, bad_kappa) + + random = Generator(MT19937(self.seed)) + actual = random.vonmises(mu, kappa * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, random.vonmises, mu, bad_kappa * 3) + + def test_pareto(self): + a = [1] + bad_a = [-1] + desired = np.array([0.95905052946317, 0.2383810889437 , 1.04988745750013]) + + random = Generator(MT19937(self.seed)) + actual = random.pareto(a * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, random.pareto, bad_a * 3) + + def test_weibull(self): + a = [1] + bad_a = [-1] + desired = np.array([0.67245993212806, 0.21380495318094, 0.7177848928629]) + + random = Generator(MT19937(self.seed)) + actual = random.weibull(a * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, random.weibull, bad_a * 3) + + def test_power(self): + a = [1] + bad_a = [-1] + desired = np.array([0.48954864361052, 0.19249412888486, 0.51216834058807]) + + random = Generator(MT19937(self.seed)) + actual = random.power(a * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, random.power, bad_a * 3) + + def test_laplace(self): + loc = [0] + scale = [1] + bad_scale = [-1] + desired = np.array([-1.09698732625119, -0.93470271947368, 0.71592671378202]) + + random = Generator(MT19937(self.seed)) + laplace = random.laplace + actual = laplace(loc * 3, scale) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, laplace, loc * 3, bad_scale) + + random = Generator(MT19937(self.seed)) + laplace = random.laplace + actual = laplace(loc, scale * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, laplace, loc, bad_scale * 3) + + def test_gumbel(self): + loc = [0] + scale = [1] + bad_scale = [-1] + desired = np.array([1.70020068231762, 1.52054354273631, -0.34293267607081]) + + random = Generator(MT19937(self.seed)) + gumbel = random.gumbel + actual = gumbel(loc * 3, scale) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, gumbel, loc * 3, bad_scale) + + random = Generator(MT19937(self.seed)) + gumbel = random.gumbel + actual = gumbel(loc, scale * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, gumbel, loc, bad_scale * 3) + + def test_logistic(self): + loc = [0] + scale = [1] + bad_scale = [-1] + desired = np.array([-1.607487640433, -1.40925686003678, 1.12887112820397]) + + random = Generator(MT19937(self.seed)) + actual = random.logistic(loc * 3, scale) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, random.logistic, loc * 3, bad_scale) + + random = Generator(MT19937(self.seed)) + actual = random.logistic(loc, scale * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, random.logistic, loc, bad_scale * 3) + assert_equal(random.logistic(1.0, 0.0), 1.0) + + def test_lognormal(self): + mean = [0] + sigma = [1] + bad_sigma = [-1] + desired = np.array([0.67884390500697, 2.21653186290321, 1.01990310084276]) + + random = Generator(MT19937(self.seed)) + lognormal = random.lognormal + actual = lognormal(mean * 3, sigma) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, lognormal, mean * 3, bad_sigma) + + random = Generator(MT19937(self.seed)) + actual = random.lognormal(mean, sigma * 3) + assert_raises(ValueError, random.lognormal, mean, bad_sigma * 3) + + def test_rayleigh(self): + scale = [1] + bad_scale = [-1] + desired = np.array( + [1.1597068009872629, + 0.6539188836253857, + 1.1981526554349398] + ) + + random = Generator(MT19937(self.seed)) + actual = random.rayleigh(scale * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, random.rayleigh, bad_scale * 3) + + def test_wald(self): + mean = [0.5] + scale = [1] + bad_mean = [0] + bad_scale = [-2] + desired = np.array([0.38052407392905, 0.50701641508592, 0.484935249864]) + + random = Generator(MT19937(self.seed)) + actual = random.wald(mean * 3, scale) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, random.wald, bad_mean * 3, scale) + assert_raises(ValueError, random.wald, mean * 3, bad_scale) + + random = Generator(MT19937(self.seed)) + actual = random.wald(mean, scale * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, random.wald, bad_mean, scale * 3) + assert_raises(ValueError, random.wald, mean, bad_scale * 3) + + def test_triangular(self): + left = [1] + right = [3] + mode = [2] + bad_left_one = [3] + bad_mode_one = [4] + bad_left_two, bad_mode_two = right * 2 + desired = np.array([1.57781954604754, 1.62665986867957, 2.30090130831326]) + + random = Generator(MT19937(self.seed)) + triangular = random.triangular + actual = triangular(left * 3, mode, right) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, triangular, bad_left_one * 3, mode, right) + assert_raises(ValueError, triangular, left * 3, bad_mode_one, right) + assert_raises(ValueError, triangular, bad_left_two * 3, bad_mode_two, + right) + + random = Generator(MT19937(self.seed)) + triangular = random.triangular + actual = triangular(left, mode * 3, right) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, triangular, bad_left_one, mode * 3, right) + assert_raises(ValueError, triangular, left, bad_mode_one * 3, right) + assert_raises(ValueError, triangular, bad_left_two, bad_mode_two * 3, + right) + + random = Generator(MT19937(self.seed)) + triangular = random.triangular + actual = triangular(left, mode, right * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, triangular, bad_left_one, mode, right * 3) + assert_raises(ValueError, triangular, left, bad_mode_one, right * 3) + assert_raises(ValueError, triangular, bad_left_two, bad_mode_two, + right * 3) + + assert_raises(ValueError, triangular, 10., 0., 20.) + assert_raises(ValueError, triangular, 10., 25., 20.) + assert_raises(ValueError, triangular, 10., 10., 10.) + + def test_binomial(self): + n = [1] + p = [0.5] + bad_n = [-1] + bad_p_one = [-1] + bad_p_two = [1.5] + desired = np.array([0, 0, 1]) + + random = Generator(MT19937(self.seed)) + binom = random.binomial + actual = binom(n * 3, p) + assert_array_equal(actual, desired) + assert_raises(ValueError, binom, bad_n * 3, p) + assert_raises(ValueError, binom, n * 3, bad_p_one) + assert_raises(ValueError, binom, n * 3, bad_p_two) + + random = Generator(MT19937(self.seed)) + actual = random.binomial(n, p * 3) + assert_array_equal(actual, desired) + assert_raises(ValueError, binom, bad_n, p * 3) + assert_raises(ValueError, binom, n, bad_p_one * 3) + assert_raises(ValueError, binom, n, bad_p_two * 3) + + def test_negative_binomial(self): + n = [1] + p = [0.5] + bad_n = [-1] + bad_p_one = [-1] + bad_p_two = [1.5] + desired = np.array([0, 2, 1], dtype=np.int64) + + random = Generator(MT19937(self.seed)) + neg_binom = random.negative_binomial + actual = neg_binom(n * 3, p) + assert_array_equal(actual, desired) + assert_raises(ValueError, neg_binom, bad_n * 3, p) + assert_raises(ValueError, neg_binom, n * 3, bad_p_one) + assert_raises(ValueError, neg_binom, n * 3, bad_p_two) + + random = Generator(MT19937(self.seed)) + neg_binom = random.negative_binomial + actual = neg_binom(n, p * 3) + assert_array_equal(actual, desired) + assert_raises(ValueError, neg_binom, bad_n, p * 3) + assert_raises(ValueError, neg_binom, n, bad_p_one * 3) + assert_raises(ValueError, neg_binom, n, bad_p_two * 3) + + def test_poisson(self): + + lam = [1] + bad_lam_one = [-1] + desired = np.array([0, 0, 3]) + + random = Generator(MT19937(self.seed)) + max_lam = random._poisson_lam_max + bad_lam_two = [max_lam * 2] + poisson = random.poisson + actual = poisson(lam * 3) + assert_array_equal(actual, desired) + assert_raises(ValueError, poisson, bad_lam_one * 3) + assert_raises(ValueError, poisson, bad_lam_two * 3) + + def test_zipf(self): + a = [2] + bad_a = [0] + desired = np.array([1, 8, 1]) + + random = Generator(MT19937(self.seed)) + zipf = random.zipf + actual = zipf(a * 3) + assert_array_equal(actual, desired) + assert_raises(ValueError, zipf, bad_a * 3) + with np.errstate(invalid='ignore'): + assert_raises(ValueError, zipf, np.nan) + assert_raises(ValueError, zipf, [0, 0, np.nan]) + + def test_geometric(self): + p = [0.5] + bad_p_one = [-1] + bad_p_two = [1.5] + desired = np.array([1, 1, 3]) + + random = Generator(MT19937(self.seed)) + geometric = random.geometric + actual = geometric(p * 3) + assert_array_equal(actual, desired) + assert_raises(ValueError, geometric, bad_p_one * 3) + assert_raises(ValueError, geometric, bad_p_two * 3) + + def test_hypergeometric(self): + ngood = [1] + nbad = [2] + nsample = [2] + bad_ngood = [-1] + bad_nbad = [-2] + bad_nsample_one = [-1] + bad_nsample_two = [4] + desired = np.array([0, 0, 1]) + + random = Generator(MT19937(self.seed)) + actual = random.hypergeometric(ngood * 3, nbad, nsample) + assert_array_equal(actual, desired) + assert_raises(ValueError, random.hypergeometric, bad_ngood * 3, nbad, nsample) + assert_raises(ValueError, random.hypergeometric, ngood * 3, bad_nbad, nsample) + assert_raises(ValueError, random.hypergeometric, ngood * 3, nbad, bad_nsample_one) + assert_raises(ValueError, random.hypergeometric, ngood * 3, nbad, bad_nsample_two) + + random = Generator(MT19937(self.seed)) + actual = random.hypergeometric(ngood, nbad * 3, nsample) + assert_array_equal(actual, desired) + assert_raises(ValueError, random.hypergeometric, bad_ngood, nbad * 3, nsample) + assert_raises(ValueError, random.hypergeometric, ngood, bad_nbad * 3, nsample) + assert_raises(ValueError, random.hypergeometric, ngood, nbad * 3, bad_nsample_one) + assert_raises(ValueError, random.hypergeometric, ngood, nbad * 3, bad_nsample_two) + + random = Generator(MT19937(self.seed)) + hypergeom = random.hypergeometric + actual = hypergeom(ngood, nbad, nsample * 3) + assert_array_equal(actual, desired) + assert_raises(ValueError, hypergeom, bad_ngood, nbad, nsample * 3) + assert_raises(ValueError, hypergeom, ngood, bad_nbad, nsample * 3) + assert_raises(ValueError, hypergeom, ngood, nbad, bad_nsample_one * 3) + assert_raises(ValueError, hypergeom, ngood, nbad, bad_nsample_two * 3) + + assert_raises(ValueError, hypergeom, -1, 10, 20) + assert_raises(ValueError, hypergeom, 10, -1, 20) + assert_raises(ValueError, hypergeom, 10, 10, -1) + assert_raises(ValueError, hypergeom, 10, 10, 25) + + # ValueError for arguments that are too big. + assert_raises(ValueError, hypergeom, 2**30, 10, 20) + assert_raises(ValueError, hypergeom, 999, 2**31, 50) + assert_raises(ValueError, hypergeom, 999, [2**29, 2**30], 1000) + + def test_logseries(self): + p = [0.5] + bad_p_one = [2] + bad_p_two = [-1] + desired = np.array([1, 1, 1]) + + random = Generator(MT19937(self.seed)) + logseries = random.logseries + actual = logseries(p * 3) + assert_array_equal(actual, desired) + assert_raises(ValueError, logseries, bad_p_one * 3) + assert_raises(ValueError, logseries, bad_p_two * 3) + + def test_multinomial(self): + random = Generator(MT19937(self.seed)) + actual = random.multinomial([5, 20], [1 / 6.] * 6, size=(3, 2)) + desired = np.array([[[0, 0, 2, 1, 2, 0], + [2, 3, 6, 4, 2, 3]], + [[1, 0, 1, 0, 2, 1], + [7, 2, 2, 1, 4, 4]], + [[0, 2, 0, 1, 2, 0], + [3, 2, 3, 3, 4, 5]]], dtype=np.int64) + assert_array_equal(actual, desired) + + random = Generator(MT19937(self.seed)) + actual = random.multinomial([5, 20], [1 / 6.] * 6) + desired = np.array([[0, 0, 2, 1, 2, 0], + [2, 3, 6, 4, 2, 3]], dtype=np.int64) + assert_array_equal(actual, desired) + + random = Generator(MT19937(self.seed)) + actual = random.multinomial([5, 20], [[1 / 6.] * 6] * 2) + desired = np.array([[0, 0, 2, 1, 2, 0], + [2, 3, 6, 4, 2, 3]], dtype=np.int64) + assert_array_equal(actual, desired) + + random = Generator(MT19937(self.seed)) + actual = random.multinomial([[5], [20]], [[1 / 6.] * 6] * 2) + desired = np.array([[[0, 0, 2, 1, 2, 0], + [0, 0, 2, 1, 1, 1]], + [[4, 2, 3, 3, 5, 3], + [7, 2, 2, 1, 4, 4]]], dtype=np.int64) + assert_array_equal(actual, desired) + + @pytest.mark.parametrize("n", [10, + np.array([10, 10]), + np.array([[[10]], [[10]]]) + ] + ) + def test_multinomial_pval_broadcast(self, n): + random = Generator(MT19937(self.seed)) + pvals = np.array([1 / 4] * 4) + actual = random.multinomial(n, pvals) + n_shape = tuple() if isinstance(n, int) else n.shape + expected_shape = n_shape + (4,) + assert actual.shape == expected_shape + pvals = np.vstack([pvals, pvals]) + actual = random.multinomial(n, pvals) + expected_shape = np.broadcast_shapes(n_shape, pvals.shape[:-1]) + (4,) + assert actual.shape == expected_shape + + pvals = np.vstack([[pvals], [pvals]]) + actual = random.multinomial(n, pvals) + expected_shape = np.broadcast_shapes(n_shape, pvals.shape[:-1]) + assert actual.shape == expected_shape + (4,) + actual = random.multinomial(n, pvals, size=(3, 2) + expected_shape) + assert actual.shape == (3, 2) + expected_shape + (4,) + + with pytest.raises(ValueError): + # Ensure that size is not broadcast + actual = random.multinomial(n, pvals, size=(1,) * 6) + + def test_invalid_pvals_broadcast(self): + random = Generator(MT19937(self.seed)) + pvals = [[1 / 6] * 6, [1 / 4] * 6] + assert_raises(ValueError, random.multinomial, 1, pvals) + assert_raises(ValueError, random.multinomial, 6, 0.5) + + def test_empty_outputs(self): + random = Generator(MT19937(self.seed)) + actual = random.multinomial(np.empty((10, 0, 6), "i8"), [1 / 6] * 6) + assert actual.shape == (10, 0, 6, 6) + actual = random.multinomial(12, np.empty((10, 0, 10))) + assert actual.shape == (10, 0, 10) + actual = random.multinomial(np.empty((3, 0, 7), "i8"), + np.empty((3, 0, 7, 4))) + assert actual.shape == (3, 0, 7, 4) + + +@pytest.mark.skipif(IS_WASM, reason="can't start thread") +class TestThread: + # make sure each state produces the same sequence even in threads + def setup_method(self): + self.seeds = range(4) + + def check_function(self, function, sz): + from threading import Thread + + out1 = np.empty((len(self.seeds),) + sz) + out2 = np.empty((len(self.seeds),) + sz) + + # threaded generation + t = [Thread(target=function, args=(Generator(MT19937(s)), o)) + for s, o in zip(self.seeds, out1)] + [x.start() for x in t] + [x.join() for x in t] + + # the same serial + for s, o in zip(self.seeds, out2): + function(Generator(MT19937(s)), o) + + # these platforms change x87 fpu precision mode in threads + if np.intp().dtype.itemsize == 4 and sys.platform == "win32": + assert_array_almost_equal(out1, out2) + else: + assert_array_equal(out1, out2) + + def test_normal(self): + def gen_random(state, out): + out[...] = state.normal(size=10000) + + self.check_function(gen_random, sz=(10000,)) + + def test_exp(self): + def gen_random(state, out): + out[...] = state.exponential(scale=np.ones((100, 1000))) + + self.check_function(gen_random, sz=(100, 1000)) + + def test_multinomial(self): + def gen_random(state, out): + out[...] = state.multinomial(10, [1 / 6.] * 6, size=10000) + + self.check_function(gen_random, sz=(10000, 6)) + + +# See Issue #4263 +class TestSingleEltArrayInput: + def setup_method(self): + self.argOne = np.array([2]) + self.argTwo = np.array([3]) + self.argThree = np.array([4]) + self.tgtShape = (1,) + + def test_one_arg_funcs(self): + funcs = (random.exponential, random.standard_gamma, + random.chisquare, random.standard_t, + random.pareto, random.weibull, + random.power, random.rayleigh, + random.poisson, random.zipf, + random.geometric, random.logseries) + + probfuncs = (random.geometric, random.logseries) + + for func in funcs: + if func in probfuncs: # p < 1.0 + out = func(np.array([0.5])) + + else: + out = func(self.argOne) + + assert_equal(out.shape, self.tgtShape) + + def test_two_arg_funcs(self): + funcs = (random.uniform, random.normal, + random.beta, random.gamma, + random.f, random.noncentral_chisquare, + random.vonmises, random.laplace, + random.gumbel, random.logistic, + random.lognormal, random.wald, + random.binomial, random.negative_binomial) + + probfuncs = (random.binomial, random.negative_binomial) + + for func in funcs: + if func in probfuncs: # p <= 1 + argTwo = np.array([0.5]) + + else: + argTwo = self.argTwo + + out = func(self.argOne, argTwo) + assert_equal(out.shape, self.tgtShape) + + out = func(self.argOne[0], argTwo) + assert_equal(out.shape, self.tgtShape) + + out = func(self.argOne, argTwo[0]) + assert_equal(out.shape, self.tgtShape) + + def test_integers(self, endpoint): + itype = [np.bool_, np.int8, np.uint8, np.int16, np.uint16, + np.int32, np.uint32, np.int64, np.uint64] + func = random.integers + high = np.array([1]) + low = np.array([0]) + + for dt in itype: + out = func(low, high, endpoint=endpoint, dtype=dt) + assert_equal(out.shape, self.tgtShape) + + out = func(low[0], high, endpoint=endpoint, dtype=dt) + assert_equal(out.shape, self.tgtShape) + + out = func(low, high[0], endpoint=endpoint, dtype=dt) + assert_equal(out.shape, self.tgtShape) + + def test_three_arg_funcs(self): + funcs = [random.noncentral_f, random.triangular, + random.hypergeometric] + + for func in funcs: + out = func(self.argOne, self.argTwo, self.argThree) + assert_equal(out.shape, self.tgtShape) + + out = func(self.argOne[0], self.argTwo, self.argThree) + assert_equal(out.shape, self.tgtShape) + + out = func(self.argOne, self.argTwo[0], self.argThree) + assert_equal(out.shape, self.tgtShape) + + +@pytest.mark.parametrize("config", JUMP_TEST_DATA) +def test_jumped(config): + # Each config contains the initial seed, a number of raw steps + # the sha256 hashes of the initial and the final states' keys and + # the position of the initial and the final state. + # These were produced using the original C implementation. + seed = config["seed"] + steps = config["steps"] + + mt19937 = MT19937(seed) + # Burn step + mt19937.random_raw(steps) + key = mt19937.state["state"]["key"] + if sys.byteorder == 'big': + key = key.byteswap() + sha256 = hashlib.sha256(key) + assert mt19937.state["state"]["pos"] == config["initial"]["pos"] + assert sha256.hexdigest() == config["initial"]["key_sha256"] + + jumped = mt19937.jumped() + key = jumped.state["state"]["key"] + if sys.byteorder == 'big': + key = key.byteswap() + sha256 = hashlib.sha256(key) + assert jumped.state["state"]["pos"] == config["jumped"]["pos"] + assert sha256.hexdigest() == config["jumped"]["key_sha256"] + + +def test_broadcast_size_error(): + mu = np.ones(3) + sigma = np.ones((4, 3)) + size = (10, 4, 2) + assert random.normal(mu, sigma, size=(5, 4, 3)).shape == (5, 4, 3) + with pytest.raises(ValueError): + random.normal(mu, sigma, size=size) + with pytest.raises(ValueError): + random.normal(mu, sigma, size=(1, 3)) + with pytest.raises(ValueError): + random.normal(mu, sigma, size=(4, 1, 1)) + # 1 arg + shape = np.ones((4, 3)) + with pytest.raises(ValueError): + random.standard_gamma(shape, size=size) + with pytest.raises(ValueError): + random.standard_gamma(shape, size=(3,)) + with pytest.raises(ValueError): + random.standard_gamma(shape, size=3) + # Check out + out = np.empty(size) + with pytest.raises(ValueError): + random.standard_gamma(shape, out=out) + + # 2 arg + with pytest.raises(ValueError): + random.binomial(1, [0.3, 0.7], size=(2, 1)) + with pytest.raises(ValueError): + random.binomial([1, 2], 0.3, size=(2, 1)) + with pytest.raises(ValueError): + random.binomial([1, 2], [0.3, 0.7], size=(2, 1)) + with pytest.raises(ValueError): + random.multinomial([2, 2], [.3, .7], size=(2, 1)) + + # 3 arg + a = random.chisquare(5, size=3) + b = random.chisquare(5, size=(4, 3)) + c = random.chisquare(5, size=(5, 4, 3)) + assert random.noncentral_f(a, b, c).shape == (5, 4, 3) + with pytest.raises(ValueError, match=r"Output size \(6, 5, 1, 1\) is"): + random.noncentral_f(a, b, c, size=(6, 5, 1, 1)) + + +def test_broadcast_size_scalar(): + mu = np.ones(3) + sigma = np.ones(3) + random.normal(mu, sigma, size=3) + with pytest.raises(ValueError): + random.normal(mu, sigma, size=2) + + +def test_ragged_shuffle(): + # GH 18142 + seq = [[], [], 1] + gen = Generator(MT19937(0)) + assert_no_warnings(gen.shuffle, seq) + assert seq == [1, [], []] + + +@pytest.mark.parametrize("high", [-2, [-2]]) +@pytest.mark.parametrize("endpoint", [True, False]) +def test_single_arg_integer_exception(high, endpoint): + # GH 14333 + gen = Generator(MT19937(0)) + msg = 'high < 0' if endpoint else 'high <= 0' + with pytest.raises(ValueError, match=msg): + gen.integers(high, endpoint=endpoint) + msg = 'low > high' if endpoint else 'low >= high' + with pytest.raises(ValueError, match=msg): + gen.integers(-1, high, endpoint=endpoint) + with pytest.raises(ValueError, match=msg): + gen.integers([-1], high, endpoint=endpoint) + + +@pytest.mark.parametrize("dtype", ["f4", "f8"]) +def test_c_contig_req_out(dtype): + # GH 18704 + out = np.empty((2, 3), order="F", dtype=dtype) + shape = [1, 2, 3] + with pytest.raises(ValueError, match="Supplied output array"): + random.standard_gamma(shape, out=out, dtype=dtype) + with pytest.raises(ValueError, match="Supplied output array"): + random.standard_gamma(shape, out=out, size=out.shape, dtype=dtype) + + +@pytest.mark.parametrize("dtype", ["f4", "f8"]) +@pytest.mark.parametrize("order", ["F", "C"]) +@pytest.mark.parametrize("dist", [random.standard_normal, random.random]) +def test_contig_req_out(dist, order, dtype): + # GH 18704 + out = np.empty((2, 3), dtype=dtype, order=order) + variates = dist(out=out, dtype=dtype) + assert variates is out + variates = dist(out=out, dtype=dtype, size=out.shape) + assert variates is out + + +def test_generator_ctor_old_style_pickle(): + rg = np.random.Generator(np.random.PCG64DXSM(0)) + rg.standard_normal(1) + # Directly call reduce which is used in pickling + ctor, args, state_a = rg.__reduce__() + # Simulate unpickling an old pickle that only has the name + assert args[:1] == ("PCG64DXSM",) + b = ctor(*args[:1]) + b.bit_generator.state = state_a + state_b = b.bit_generator.state + assert state_a == state_b diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/test_generator_mt19937_regressions.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/test_generator_mt19937_regressions.py new file mode 100644 index 0000000000000000000000000000000000000000..f16af2b293ce21642c32e90af6f3ed22476158e6 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/test_generator_mt19937_regressions.py @@ -0,0 +1,165 @@ +from numpy.testing import (assert_, assert_array_equal) +import numpy as np +import pytest +from numpy.random import Generator, MT19937 + + +class TestRegression: + + def setup_method(self): + self.mt19937 = Generator(MT19937(121263137472525314065)) + + def test_vonmises_range(self): + # Make sure generated random variables are in [-pi, pi]. + # Regression test for ticket #986. + for mu in np.linspace(-7., 7., 5): + r = self.mt19937.vonmises(mu, 1, 50) + assert_(np.all(r > -np.pi) and np.all(r <= np.pi)) + + def test_hypergeometric_range(self): + # Test for ticket #921 + assert_(np.all(self.mt19937.hypergeometric(3, 18, 11, size=10) < 4)) + assert_(np.all(self.mt19937.hypergeometric(18, 3, 11, size=10) > 0)) + + # Test for ticket #5623 + args = (2**20 - 2, 2**20 - 2, 2**20 - 2) # Check for 32-bit systems + assert_(self.mt19937.hypergeometric(*args) > 0) + + def test_logseries_convergence(self): + # Test for ticket #923 + N = 1000 + rvsn = self.mt19937.logseries(0.8, size=N) + # these two frequency counts should be close to theoretical + # numbers with this large sample + # theoretical large N result is 0.49706795 + freq = np.sum(rvsn == 1) / N + msg = f'Frequency was {freq:f}, should be > 0.45' + assert_(freq > 0.45, msg) + # theoretical large N result is 0.19882718 + freq = np.sum(rvsn == 2) / N + msg = f'Frequency was {freq:f}, should be < 0.23' + assert_(freq < 0.23, msg) + + def test_shuffle_mixed_dimension(self): + # Test for trac ticket #2074 + for t in [[1, 2, 3, None], + [(1, 1), (2, 2), (3, 3), None], + [1, (2, 2), (3, 3), None], + [(1, 1), 2, 3, None]]: + mt19937 = Generator(MT19937(12345)) + shuffled = np.array(t, dtype=object) + mt19937.shuffle(shuffled) + expected = np.array([t[2], t[0], t[3], t[1]], dtype=object) + assert_array_equal(np.array(shuffled, dtype=object), expected) + + def test_call_within_randomstate(self): + # Check that custom BitGenerator does not call into global state + res = np.array([1, 8, 0, 1, 5, 3, 3, 8, 1, 4]) + for i in range(3): + mt19937 = Generator(MT19937(i)) + m = Generator(MT19937(4321)) + # If m.state is not honored, the result will change + assert_array_equal(m.choice(10, size=10, p=np.ones(10)/10.), res) + + def test_multivariate_normal_size_types(self): + # Test for multivariate_normal issue with 'size' argument. + # Check that the multivariate_normal size argument can be a + # numpy integer. + self.mt19937.multivariate_normal([0], [[0]], size=1) + self.mt19937.multivariate_normal([0], [[0]], size=np.int_(1)) + self.mt19937.multivariate_normal([0], [[0]], size=np.int64(1)) + + def test_beta_small_parameters(self): + # Test that beta with small a and b parameters does not produce + # NaNs due to roundoff errors causing 0 / 0, gh-5851 + x = self.mt19937.beta(0.0001, 0.0001, size=100) + assert_(not np.any(np.isnan(x)), 'Nans in mt19937.beta') + + def test_beta_very_small_parameters(self): + # gh-24203: beta would hang with very small parameters. + self.mt19937.beta(1e-49, 1e-40) + + def test_beta_ridiculously_small_parameters(self): + # gh-24266: beta would generate nan when the parameters + # were subnormal or a small multiple of the smallest normal. + tiny = np.finfo(1.0).tiny + x = self.mt19937.beta(tiny/32, tiny/40, size=50) + assert not np.any(np.isnan(x)) + + def test_choice_sum_of_probs_tolerance(self): + # The sum of probs should be 1.0 with some tolerance. + # For low precision dtypes the tolerance was too tight. + # See numpy github issue 6123. + a = [1, 2, 3] + counts = [4, 4, 2] + for dt in np.float16, np.float32, np.float64: + probs = np.array(counts, dtype=dt) / sum(counts) + c = self.mt19937.choice(a, p=probs) + assert_(c in a) + with pytest.raises(ValueError): + self.mt19937.choice(a, p=probs*0.9) + + def test_shuffle_of_array_of_different_length_strings(self): + # Test that permuting an array of different length strings + # will not cause a segfault on garbage collection + # Tests gh-7710 + + a = np.array(['a', 'a' * 1000]) + + for _ in range(100): + self.mt19937.shuffle(a) + + # Force Garbage Collection - should not segfault. + import gc + gc.collect() + + def test_shuffle_of_array_of_objects(self): + # Test that permuting an array of objects will not cause + # a segfault on garbage collection. + # See gh-7719 + a = np.array([np.arange(1), np.arange(4)], dtype=object) + + for _ in range(1000): + self.mt19937.shuffle(a) + + # Force Garbage Collection - should not segfault. + import gc + gc.collect() + + def test_permutation_subclass(self): + + class N(np.ndarray): + pass + + mt19937 = Generator(MT19937(1)) + orig = np.arange(3).view(N) + perm = mt19937.permutation(orig) + assert_array_equal(perm, np.array([2, 0, 1])) + assert_array_equal(orig, np.arange(3).view(N)) + + class M: + a = np.arange(5) + + def __array__(self): + return self.a + + mt19937 = Generator(MT19937(1)) + m = M() + perm = mt19937.permutation(m) + assert_array_equal(perm, np.array([4, 1, 3, 0, 2])) + assert_array_equal(m.__array__(), np.arange(5)) + + def test_gamma_0(self): + assert self.mt19937.standard_gamma(0.0) == 0.0 + assert_array_equal(self.mt19937.standard_gamma([0.0]), 0.0) + + actual = self.mt19937.standard_gamma([0.0], dtype='float') + expected = np.array([0.], dtype=np.float32) + assert_array_equal(actual, expected) + + def test_geometric_tiny_prob(self): + # Regression test for gh-17007. + # When p = 1e-30, the probability that a sample will exceed 2**63-1 + # is 0.9999999999907766, so we expect the result to be all 2**63-1. + assert_array_equal(self.mt19937.geometric(p=1e-30, size=3), + np.iinfo(np.int64).max) diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/test_random.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/test_random.py new file mode 100644 index 0000000000000000000000000000000000000000..3d081fe1dbd1c868fe022480330711024804ca20 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/test_random.py @@ -0,0 +1,1750 @@ +import warnings + +import pytest + +import numpy as np +from numpy.testing import ( + assert_, assert_raises, assert_equal, assert_warns, + assert_no_warnings, assert_array_equal, assert_array_almost_equal, + suppress_warnings, IS_WASM + ) +from numpy import random +import sys + + +class TestSeed: + def test_scalar(self): + s = np.random.RandomState(0) + assert_equal(s.randint(1000), 684) + s = np.random.RandomState(4294967295) + assert_equal(s.randint(1000), 419) + + def test_array(self): + s = np.random.RandomState(range(10)) + assert_equal(s.randint(1000), 468) + s = np.random.RandomState(np.arange(10)) + assert_equal(s.randint(1000), 468) + s = np.random.RandomState([0]) + assert_equal(s.randint(1000), 973) + s = np.random.RandomState([4294967295]) + assert_equal(s.randint(1000), 265) + + def test_invalid_scalar(self): + # seed must be an unsigned 32 bit integer + assert_raises(TypeError, np.random.RandomState, -0.5) + assert_raises(ValueError, np.random.RandomState, -1) + + def test_invalid_array(self): + # seed must be an unsigned 32 bit integer + assert_raises(TypeError, np.random.RandomState, [-0.5]) + assert_raises(ValueError, np.random.RandomState, [-1]) + assert_raises(ValueError, np.random.RandomState, [4294967296]) + assert_raises(ValueError, np.random.RandomState, [1, 2, 4294967296]) + assert_raises(ValueError, np.random.RandomState, [1, -2, 4294967296]) + + def test_invalid_array_shape(self): + # gh-9832 + assert_raises(ValueError, np.random.RandomState, + np.array([], dtype=np.int64)) + assert_raises(ValueError, np.random.RandomState, [[1, 2, 3]]) + assert_raises(ValueError, np.random.RandomState, [[1, 2, 3], + [4, 5, 6]]) + + +class TestBinomial: + def test_n_zero(self): + # Tests the corner case of n == 0 for the binomial distribution. + # binomial(0, p) should be zero for any p in [0, 1]. + # This test addresses issue #3480. + zeros = np.zeros(2, dtype='int') + for p in [0, .5, 1]: + assert_(random.binomial(0, p) == 0) + assert_array_equal(random.binomial(zeros, p), zeros) + + def test_p_is_nan(self): + # Issue #4571. + assert_raises(ValueError, random.binomial, 1, np.nan) + + +class TestMultinomial: + def test_basic(self): + random.multinomial(100, [0.2, 0.8]) + + def test_zero_probability(self): + random.multinomial(100, [0.2, 0.8, 0.0, 0.0, 0.0]) + + def test_int_negative_interval(self): + assert_(-5 <= random.randint(-5, -1) < -1) + x = random.randint(-5, -1, 5) + assert_(np.all(-5 <= x)) + assert_(np.all(x < -1)) + + def test_size(self): + # gh-3173 + p = [0.5, 0.5] + assert_equal(np.random.multinomial(1, p, np.uint32(1)).shape, (1, 2)) + assert_equal(np.random.multinomial(1, p, np.uint32(1)).shape, (1, 2)) + assert_equal(np.random.multinomial(1, p, np.uint32(1)).shape, (1, 2)) + assert_equal(np.random.multinomial(1, p, [2, 2]).shape, (2, 2, 2)) + assert_equal(np.random.multinomial(1, p, (2, 2)).shape, (2, 2, 2)) + assert_equal(np.random.multinomial(1, p, np.array((2, 2))).shape, + (2, 2, 2)) + + assert_raises(TypeError, np.random.multinomial, 1, p, + float(1)) + + def test_multidimensional_pvals(self): + assert_raises(ValueError, np.random.multinomial, 10, [[0, 1]]) + assert_raises(ValueError, np.random.multinomial, 10, [[0], [1]]) + assert_raises(ValueError, np.random.multinomial, 10, [[[0], [1]], [[1], [0]]]) + assert_raises(ValueError, np.random.multinomial, 10, np.array([[0, 1], [1, 0]])) + + +class TestSetState: + def setup_method(self): + self.seed = 1234567890 + self.prng = random.RandomState(self.seed) + self.state = self.prng.get_state() + + def test_basic(self): + old = self.prng.tomaxint(16) + self.prng.set_state(self.state) + new = self.prng.tomaxint(16) + assert_(np.all(old == new)) + + def test_gaussian_reset(self): + # Make sure the cached every-other-Gaussian is reset. + old = self.prng.standard_normal(size=3) + self.prng.set_state(self.state) + new = self.prng.standard_normal(size=3) + assert_(np.all(old == new)) + + def test_gaussian_reset_in_media_res(self): + # When the state is saved with a cached Gaussian, make sure the + # cached Gaussian is restored. + + self.prng.standard_normal() + state = self.prng.get_state() + old = self.prng.standard_normal(size=3) + self.prng.set_state(state) + new = self.prng.standard_normal(size=3) + assert_(np.all(old == new)) + + def test_backwards_compatibility(self): + # Make sure we can accept old state tuples that do not have the + # cached Gaussian value. + old_state = self.state[:-2] + x1 = self.prng.standard_normal(size=16) + self.prng.set_state(old_state) + x2 = self.prng.standard_normal(size=16) + self.prng.set_state(self.state) + x3 = self.prng.standard_normal(size=16) + assert_(np.all(x1 == x2)) + assert_(np.all(x1 == x3)) + + def test_negative_binomial(self): + # Ensure that the negative binomial results take floating point + # arguments without truncation. + self.prng.negative_binomial(0.5, 0.5) + + def test_set_invalid_state(self): + # gh-25402 + with pytest.raises(IndexError): + self.prng.set_state(()) + + +class TestRandint: + + rfunc = np.random.randint + + # valid integer/boolean types + itype = [np.bool_, np.int8, np.uint8, np.int16, np.uint16, + np.int32, np.uint32, np.int64, np.uint64] + + def test_unsupported_type(self): + assert_raises(TypeError, self.rfunc, 1, dtype=float) + + def test_bounds_checking(self): + for dt in self.itype: + lbnd = 0 if dt is np.bool_ else np.iinfo(dt).min + ubnd = 2 if dt is np.bool_ else np.iinfo(dt).max + 1 + assert_raises(ValueError, self.rfunc, lbnd - 1, ubnd, dtype=dt) + assert_raises(ValueError, self.rfunc, lbnd, ubnd + 1, dtype=dt) + assert_raises(ValueError, self.rfunc, ubnd, lbnd, dtype=dt) + assert_raises(ValueError, self.rfunc, 1, 0, dtype=dt) + + def test_rng_zero_and_extremes(self): + for dt in self.itype: + lbnd = 0 if dt is np.bool_ else np.iinfo(dt).min + ubnd = 2 if dt is np.bool_ else np.iinfo(dt).max + 1 + + tgt = ubnd - 1 + assert_equal(self.rfunc(tgt, tgt + 1, size=1000, dtype=dt), tgt) + + tgt = lbnd + assert_equal(self.rfunc(tgt, tgt + 1, size=1000, dtype=dt), tgt) + + tgt = (lbnd + ubnd)//2 + assert_equal(self.rfunc(tgt, tgt + 1, size=1000, dtype=dt), tgt) + + def test_full_range(self): + # Test for ticket #1690 + + for dt in self.itype: + lbnd = 0 if dt is np.bool_ else np.iinfo(dt).min + ubnd = 2 if dt is np.bool_ else np.iinfo(dt).max + 1 + + try: + self.rfunc(lbnd, ubnd, dtype=dt) + except Exception as e: + raise AssertionError("No error should have been raised, " + "but one was with the following " + "message:\n\n%s" % str(e)) + + def test_in_bounds_fuzz(self): + # Don't use fixed seed + np.random.seed() + + for dt in self.itype[1:]: + for ubnd in [4, 8, 16]: + vals = self.rfunc(2, ubnd, size=2**16, dtype=dt) + assert_(vals.max() < ubnd) + assert_(vals.min() >= 2) + + vals = self.rfunc(0, 2, size=2**16, dtype=np.bool_) + + assert_(vals.max() < 2) + assert_(vals.min() >= 0) + + def test_repeatability(self): + import hashlib + # We use a sha256 hash of generated sequences of 1000 samples + # in the range [0, 6) for all but bool, where the range + # is [0, 2). Hashes are for little endian numbers. + tgt = {'bool': '509aea74d792fb931784c4b0135392c65aec64beee12b0cc167548a2c3d31e71', + 'int16': '7b07f1a920e46f6d0fe02314155a2330bcfd7635e708da50e536c5ebb631a7d4', + 'int32': 'e577bfed6c935de944424667e3da285012e741892dcb7051a8f1ce68ab05c92f', + 'int64': '0fbead0b06759df2cfb55e43148822d4a1ff953c7eb19a5b08445a63bb64fa9e', + 'int8': '001aac3a5acb935a9b186cbe14a1ca064b8bb2dd0b045d48abeacf74d0203404', + 'uint16': '7b07f1a920e46f6d0fe02314155a2330bcfd7635e708da50e536c5ebb631a7d4', + 'uint32': 'e577bfed6c935de944424667e3da285012e741892dcb7051a8f1ce68ab05c92f', + 'uint64': '0fbead0b06759df2cfb55e43148822d4a1ff953c7eb19a5b08445a63bb64fa9e', + 'uint8': '001aac3a5acb935a9b186cbe14a1ca064b8bb2dd0b045d48abeacf74d0203404'} + + for dt in self.itype[1:]: + np.random.seed(1234) + + # view as little endian for hash + if sys.byteorder == 'little': + val = self.rfunc(0, 6, size=1000, dtype=dt) + else: + val = self.rfunc(0, 6, size=1000, dtype=dt).byteswap() + + res = hashlib.sha256(val.view(np.int8)).hexdigest() + assert_(tgt[np.dtype(dt).name] == res) + + # bools do not depend on endianness + np.random.seed(1234) + val = self.rfunc(0, 2, size=1000, dtype=bool).view(np.int8) + res = hashlib.sha256(val).hexdigest() + assert_(tgt[np.dtype(bool).name] == res) + + def test_int64_uint64_corner_case(self): + # When stored in Numpy arrays, `lbnd` is casted + # as np.int64, and `ubnd` is casted as np.uint64. + # Checking whether `lbnd` >= `ubnd` used to be + # done solely via direct comparison, which is incorrect + # because when Numpy tries to compare both numbers, + # it casts both to np.float64 because there is + # no integer superset of np.int64 and np.uint64. However, + # `ubnd` is too large to be represented in np.float64, + # causing it be round down to np.iinfo(np.int64).max, + # leading to a ValueError because `lbnd` now equals + # the new `ubnd`. + + dt = np.int64 + tgt = np.iinfo(np.int64).max + lbnd = np.int64(np.iinfo(np.int64).max) + ubnd = np.uint64(np.iinfo(np.int64).max + 1) + + # None of these function calls should + # generate a ValueError now. + actual = np.random.randint(lbnd, ubnd, dtype=dt) + assert_equal(actual, tgt) + + def test_respect_dtype_singleton(self): + # See gh-7203 + for dt in self.itype: + lbnd = 0 if dt is np.bool_ else np.iinfo(dt).min + ubnd = 2 if dt is np.bool_ else np.iinfo(dt).max + 1 + + sample = self.rfunc(lbnd, ubnd, dtype=dt) + assert_equal(sample.dtype, np.dtype(dt)) + + for dt in (bool, int): + lbnd = 0 if dt is bool else np.iinfo(dt).min + ubnd = 2 if dt is bool else np.iinfo(dt).max + 1 + + # gh-7284: Ensure that we get Python data types + sample = self.rfunc(lbnd, ubnd, dtype=dt) + assert_(not hasattr(sample, 'dtype')) + assert_equal(type(sample), dt) + + +class TestRandomDist: + # Make sure the random distribution returns the correct value for a + # given seed + + def setup_method(self): + self.seed = 1234567890 + + def test_rand(self): + np.random.seed(self.seed) + actual = np.random.rand(3, 2) + desired = np.array([[0.61879477158567997, 0.59162362775974664], + [0.88868358904449662, 0.89165480011560816], + [0.4575674820298663, 0.7781880808593471]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_randn(self): + np.random.seed(self.seed) + actual = np.random.randn(3, 2) + desired = np.array([[1.34016345771863121, 1.73759122771936081], + [1.498988344300628, -0.2286433324536169], + [2.031033998682787, 2.17032494605655257]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_randint(self): + np.random.seed(self.seed) + actual = np.random.randint(-99, 99, size=(3, 2)) + desired = np.array([[31, 3], + [-52, 41], + [-48, -66]]) + assert_array_equal(actual, desired) + + def test_random_integers(self): + np.random.seed(self.seed) + with suppress_warnings() as sup: + w = sup.record(DeprecationWarning) + actual = np.random.random_integers(-99, 99, size=(3, 2)) + assert_(len(w) == 1) + desired = np.array([[31, 3], + [-52, 41], + [-48, -66]]) + assert_array_equal(actual, desired) + + def test_random_integers_max_int(self): + # Tests whether random_integers can generate the + # maximum allowed Python int that can be converted + # into a C long. Previous implementations of this + # method have thrown an OverflowError when attempting + # to generate this integer. + with suppress_warnings() as sup: + w = sup.record(DeprecationWarning) + actual = np.random.random_integers(np.iinfo('l').max, + np.iinfo('l').max) + assert_(len(w) == 1) + + desired = np.iinfo('l').max + assert_equal(actual, desired) + + def test_random_integers_deprecated(self): + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + + # DeprecationWarning raised with high == None + assert_raises(DeprecationWarning, + np.random.random_integers, + np.iinfo('l').max) + + # DeprecationWarning raised with high != None + assert_raises(DeprecationWarning, + np.random.random_integers, + np.iinfo('l').max, np.iinfo('l').max) + + def test_random(self): + np.random.seed(self.seed) + actual = np.random.random((3, 2)) + desired = np.array([[0.61879477158567997, 0.59162362775974664], + [0.88868358904449662, 0.89165480011560816], + [0.4575674820298663, 0.7781880808593471]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_choice_uniform_replace(self): + np.random.seed(self.seed) + actual = np.random.choice(4, 4) + desired = np.array([2, 3, 2, 3]) + assert_array_equal(actual, desired) + + def test_choice_nonuniform_replace(self): + np.random.seed(self.seed) + actual = np.random.choice(4, 4, p=[0.4, 0.4, 0.1, 0.1]) + desired = np.array([1, 1, 2, 2]) + assert_array_equal(actual, desired) + + def test_choice_uniform_noreplace(self): + np.random.seed(self.seed) + actual = np.random.choice(4, 3, replace=False) + desired = np.array([0, 1, 3]) + assert_array_equal(actual, desired) + + def test_choice_nonuniform_noreplace(self): + np.random.seed(self.seed) + actual = np.random.choice(4, 3, replace=False, + p=[0.1, 0.3, 0.5, 0.1]) + desired = np.array([2, 3, 1]) + assert_array_equal(actual, desired) + + def test_choice_noninteger(self): + np.random.seed(self.seed) + actual = np.random.choice(['a', 'b', 'c', 'd'], 4) + desired = np.array(['c', 'd', 'c', 'd']) + assert_array_equal(actual, desired) + + def test_choice_exceptions(self): + sample = np.random.choice + assert_raises(ValueError, sample, -1, 3) + assert_raises(ValueError, sample, 3., 3) + assert_raises(ValueError, sample, [[1, 2], [3, 4]], 3) + assert_raises(ValueError, sample, [], 3) + assert_raises(ValueError, sample, [1, 2, 3, 4], 3, + p=[[0.25, 0.25], [0.25, 0.25]]) + assert_raises(ValueError, sample, [1, 2], 3, p=[0.4, 0.4, 0.2]) + assert_raises(ValueError, sample, [1, 2], 3, p=[1.1, -0.1]) + assert_raises(ValueError, sample, [1, 2], 3, p=[0.4, 0.4]) + assert_raises(ValueError, sample, [1, 2, 3], 4, replace=False) + # gh-13087 + assert_raises(ValueError, sample, [1, 2, 3], -2, replace=False) + assert_raises(ValueError, sample, [1, 2, 3], (-1,), replace=False) + assert_raises(ValueError, sample, [1, 2, 3], (-1, 1), replace=False) + assert_raises(ValueError, sample, [1, 2, 3], 2, + replace=False, p=[1, 0, 0]) + + def test_choice_return_shape(self): + p = [0.1, 0.9] + # Check scalar + assert_(np.isscalar(np.random.choice(2, replace=True))) + assert_(np.isscalar(np.random.choice(2, replace=False))) + assert_(np.isscalar(np.random.choice(2, replace=True, p=p))) + assert_(np.isscalar(np.random.choice(2, replace=False, p=p))) + assert_(np.isscalar(np.random.choice([1, 2], replace=True))) + assert_(np.random.choice([None], replace=True) is None) + a = np.array([1, 2]) + arr = np.empty(1, dtype=object) + arr[0] = a + assert_(np.random.choice(arr, replace=True) is a) + + # Check 0-d array + s = tuple() + assert_(not np.isscalar(np.random.choice(2, s, replace=True))) + assert_(not np.isscalar(np.random.choice(2, s, replace=False))) + assert_(not np.isscalar(np.random.choice(2, s, replace=True, p=p))) + assert_(not np.isscalar(np.random.choice(2, s, replace=False, p=p))) + assert_(not np.isscalar(np.random.choice([1, 2], s, replace=True))) + assert_(np.random.choice([None], s, replace=True).ndim == 0) + a = np.array([1, 2]) + arr = np.empty(1, dtype=object) + arr[0] = a + assert_(np.random.choice(arr, s, replace=True).item() is a) + + # Check multi dimensional array + s = (2, 3) + p = [0.1, 0.1, 0.1, 0.1, 0.4, 0.2] + assert_equal(np.random.choice(6, s, replace=True).shape, s) + assert_equal(np.random.choice(6, s, replace=False).shape, s) + assert_equal(np.random.choice(6, s, replace=True, p=p).shape, s) + assert_equal(np.random.choice(6, s, replace=False, p=p).shape, s) + assert_equal(np.random.choice(np.arange(6), s, replace=True).shape, s) + + # Check zero-size + assert_equal(np.random.randint(0, 0, size=(3, 0, 4)).shape, (3, 0, 4)) + assert_equal(np.random.randint(0, -10, size=0).shape, (0,)) + assert_equal(np.random.randint(10, 10, size=0).shape, (0,)) + assert_equal(np.random.choice(0, size=0).shape, (0,)) + assert_equal(np.random.choice([], size=(0,)).shape, (0,)) + assert_equal(np.random.choice(['a', 'b'], size=(3, 0, 4)).shape, + (3, 0, 4)) + assert_raises(ValueError, np.random.choice, [], 10) + + def test_choice_nan_probabilities(self): + a = np.array([42, 1, 2]) + p = [None, None, None] + assert_raises(ValueError, np.random.choice, a, p=p) + + def test_bytes(self): + np.random.seed(self.seed) + actual = np.random.bytes(10) + desired = b'\x82Ui\x9e\xff\x97+Wf\xa5' + assert_equal(actual, desired) + + def test_shuffle(self): + # Test lists, arrays (of various dtypes), and multidimensional versions + # of both, c-contiguous or not: + for conv in [lambda x: np.array([]), + lambda x: x, + lambda x: np.asarray(x).astype(np.int8), + lambda x: np.asarray(x).astype(np.float32), + lambda x: np.asarray(x).astype(np.complex64), + lambda x: np.asarray(x).astype(object), + lambda x: [(i, i) for i in x], + lambda x: np.asarray([[i, i] for i in x]), + lambda x: np.vstack([x, x]).T, + # gh-11442 + lambda x: (np.asarray([(i, i) for i in x], + [("a", int), ("b", int)]) + .view(np.recarray)), + # gh-4270 + lambda x: np.asarray([(i, i) for i in x], + [("a", object), ("b", np.int32)])]: + np.random.seed(self.seed) + alist = conv([1, 2, 3, 4, 5, 6, 7, 8, 9, 0]) + np.random.shuffle(alist) + actual = alist + desired = conv([0, 1, 9, 6, 2, 4, 5, 8, 7, 3]) + assert_array_equal(actual, desired) + + def test_shuffle_masked(self): + # gh-3263 + a = np.ma.masked_values(np.reshape(range(20), (5, 4)) % 3 - 1, -1) + b = np.ma.masked_values(np.arange(20) % 3 - 1, -1) + a_orig = a.copy() + b_orig = b.copy() + for i in range(50): + np.random.shuffle(a) + assert_equal( + sorted(a.data[~a.mask]), sorted(a_orig.data[~a_orig.mask])) + np.random.shuffle(b) + assert_equal( + sorted(b.data[~b.mask]), sorted(b_orig.data[~b_orig.mask])) + + @pytest.mark.parametrize("random", + [np.random, np.random.RandomState(), np.random.default_rng()]) + def test_shuffle_untyped_warning(self, random): + # Create a dict works like a sequence but isn't one + values = {0: 0, 1: 1, 2: 2, 3: 3, 4: 4, 5: 5, 6: 6} + with pytest.warns(UserWarning, + match="you are shuffling a 'dict' object") as rec: + random.shuffle(values) + assert "test_random" in rec[0].filename + + @pytest.mark.parametrize("random", + [np.random, np.random.RandomState(), np.random.default_rng()]) + @pytest.mark.parametrize("use_array_like", [True, False]) + def test_shuffle_no_object_unpacking(self, random, use_array_like): + class MyArr(np.ndarray): + pass + + items = [ + None, np.array([3]), np.float64(3), np.array(10), np.float64(7) + ] + arr = np.array(items, dtype=object) + item_ids = {id(i) for i in items} + if use_array_like: + arr = arr.view(MyArr) + + # The array was created fine, and did not modify any objects: + assert all(id(i) in item_ids for i in arr) + + if use_array_like and not isinstance(random, np.random.Generator): + # The old API gives incorrect results, but warns about it. + with pytest.warns(UserWarning, + match="Shuffling a one dimensional array.*"): + random.shuffle(arr) + else: + random.shuffle(arr) + assert all(id(i) in item_ids for i in arr) + + def test_shuffle_memoryview(self): + # gh-18273 + # allow graceful handling of memoryviews + # (treat the same as arrays) + np.random.seed(self.seed) + a = np.arange(5).data + np.random.shuffle(a) + assert_equal(np.asarray(a), [0, 1, 4, 3, 2]) + rng = np.random.RandomState(self.seed) + rng.shuffle(a) + assert_equal(np.asarray(a), [0, 1, 2, 3, 4]) + rng = np.random.default_rng(self.seed) + rng.shuffle(a) + assert_equal(np.asarray(a), [4, 1, 0, 3, 2]) + + def test_shuffle_not_writeable(self): + a = np.zeros(3) + a.flags.writeable = False + with pytest.raises(ValueError, match='read-only'): + np.random.shuffle(a) + + def test_beta(self): + np.random.seed(self.seed) + actual = np.random.beta(.1, .9, size=(3, 2)) + desired = np.array( + [[1.45341850513746058e-02, 5.31297615662868145e-04], + [1.85366619058432324e-06, 4.19214516800110563e-03], + [1.58405155108498093e-04, 1.26252891949397652e-04]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_binomial(self): + np.random.seed(self.seed) + actual = np.random.binomial(100, .456, size=(3, 2)) + desired = np.array([[37, 43], + [42, 48], + [46, 45]]) + assert_array_equal(actual, desired) + + def test_chisquare(self): + np.random.seed(self.seed) + actual = np.random.chisquare(50, size=(3, 2)) + desired = np.array([[63.87858175501090585, 68.68407748911370447], + [65.77116116901505904, 47.09686762438974483], + [72.3828403199695174, 74.18408615260374006]]) + assert_array_almost_equal(actual, desired, decimal=13) + + def test_dirichlet(self): + np.random.seed(self.seed) + alpha = np.array([51.72840233779265162, 39.74494232180943953]) + actual = np.random.mtrand.dirichlet(alpha, size=(3, 2)) + desired = np.array([[[0.54539444573611562, 0.45460555426388438], + [0.62345816822039413, 0.37654183177960598]], + [[0.55206000085785778, 0.44793999914214233], + [0.58964023305154301, 0.41035976694845688]], + [[0.59266909280647828, 0.40733090719352177], + [0.56974431743975207, 0.43025568256024799]]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_dirichlet_size(self): + # gh-3173 + p = np.array([51.72840233779265162, 39.74494232180943953]) + assert_equal(np.random.dirichlet(p, np.uint32(1)).shape, (1, 2)) + assert_equal(np.random.dirichlet(p, np.uint32(1)).shape, (1, 2)) + assert_equal(np.random.dirichlet(p, np.uint32(1)).shape, (1, 2)) + assert_equal(np.random.dirichlet(p, [2, 2]).shape, (2, 2, 2)) + assert_equal(np.random.dirichlet(p, (2, 2)).shape, (2, 2, 2)) + assert_equal(np.random.dirichlet(p, np.array((2, 2))).shape, (2, 2, 2)) + + assert_raises(TypeError, np.random.dirichlet, p, float(1)) + + def test_dirichlet_bad_alpha(self): + # gh-2089 + alpha = np.array([5.4e-01, -1.0e-16]) + assert_raises(ValueError, np.random.mtrand.dirichlet, alpha) + + # gh-15876 + assert_raises(ValueError, random.dirichlet, [[5, 1]]) + assert_raises(ValueError, random.dirichlet, [[5], [1]]) + assert_raises(ValueError, random.dirichlet, [[[5], [1]], [[1], [5]]]) + assert_raises(ValueError, random.dirichlet, np.array([[5, 1], [1, 5]])) + + def test_exponential(self): + np.random.seed(self.seed) + actual = np.random.exponential(1.1234, size=(3, 2)) + desired = np.array([[1.08342649775011624, 1.00607889924557314], + [2.46628830085216721, 2.49668106809923884], + [0.68717433461363442, 1.69175666993575979]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_exponential_0(self): + assert_equal(np.random.exponential(scale=0), 0) + assert_raises(ValueError, np.random.exponential, scale=-0.) + + def test_f(self): + np.random.seed(self.seed) + actual = np.random.f(12, 77, size=(3, 2)) + desired = np.array([[1.21975394418575878, 1.75135759791559775], + [1.44803115017146489, 1.22108959480396262], + [1.02176975757740629, 1.34431827623300415]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_gamma(self): + np.random.seed(self.seed) + actual = np.random.gamma(5, 3, size=(3, 2)) + desired = np.array([[24.60509188649287182, 28.54993563207210627], + [26.13476110204064184, 12.56988482927716078], + [31.71863275789960568, 33.30143302795922011]]) + assert_array_almost_equal(actual, desired, decimal=14) + + def test_gamma_0(self): + assert_equal(np.random.gamma(shape=0, scale=0), 0) + assert_raises(ValueError, np.random.gamma, shape=-0., scale=-0.) + + def test_geometric(self): + np.random.seed(self.seed) + actual = np.random.geometric(.123456789, size=(3, 2)) + desired = np.array([[8, 7], + [17, 17], + [5, 12]]) + assert_array_equal(actual, desired) + + def test_gumbel(self): + np.random.seed(self.seed) + actual = np.random.gumbel(loc=.123456789, scale=2.0, size=(3, 2)) + desired = np.array([[0.19591898743416816, 0.34405539668096674], + [-1.4492522252274278, -1.47374816298446865], + [1.10651090478803416, -0.69535848626236174]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_gumbel_0(self): + assert_equal(np.random.gumbel(scale=0), 0) + assert_raises(ValueError, np.random.gumbel, scale=-0.) + + def test_hypergeometric(self): + np.random.seed(self.seed) + actual = np.random.hypergeometric(10, 5, 14, size=(3, 2)) + desired = np.array([[10, 10], + [10, 10], + [9, 9]]) + assert_array_equal(actual, desired) + + # Test nbad = 0 + actual = np.random.hypergeometric(5, 0, 3, size=4) + desired = np.array([3, 3, 3, 3]) + assert_array_equal(actual, desired) + + actual = np.random.hypergeometric(15, 0, 12, size=4) + desired = np.array([12, 12, 12, 12]) + assert_array_equal(actual, desired) + + # Test ngood = 0 + actual = np.random.hypergeometric(0, 5, 3, size=4) + desired = np.array([0, 0, 0, 0]) + assert_array_equal(actual, desired) + + actual = np.random.hypergeometric(0, 15, 12, size=4) + desired = np.array([0, 0, 0, 0]) + assert_array_equal(actual, desired) + + def test_laplace(self): + np.random.seed(self.seed) + actual = np.random.laplace(loc=.123456789, scale=2.0, size=(3, 2)) + desired = np.array([[0.66599721112760157, 0.52829452552221945], + [3.12791959514407125, 3.18202813572992005], + [-0.05391065675859356, 1.74901336242837324]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_laplace_0(self): + assert_equal(np.random.laplace(scale=0), 0) + assert_raises(ValueError, np.random.laplace, scale=-0.) + + def test_logistic(self): + np.random.seed(self.seed) + actual = np.random.logistic(loc=.123456789, scale=2.0, size=(3, 2)) + desired = np.array([[1.09232835305011444, 0.8648196662399954], + [4.27818590694950185, 4.33897006346929714], + [-0.21682183359214885, 2.63373365386060332]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_lognormal(self): + np.random.seed(self.seed) + actual = np.random.lognormal(mean=.123456789, sigma=2.0, size=(3, 2)) + desired = np.array([[16.50698631688883822, 36.54846706092654784], + [22.67886599981281748, 0.71617561058995771], + [65.72798501792723869, 86.84341601437161273]]) + assert_array_almost_equal(actual, desired, decimal=13) + + def test_lognormal_0(self): + assert_equal(np.random.lognormal(sigma=0), 1) + assert_raises(ValueError, np.random.lognormal, sigma=-0.) + + def test_logseries(self): + np.random.seed(self.seed) + actual = np.random.logseries(p=.923456789, size=(3, 2)) + desired = np.array([[2, 2], + [6, 17], + [3, 6]]) + assert_array_equal(actual, desired) + + def test_multinomial(self): + np.random.seed(self.seed) + actual = np.random.multinomial(20, [1/6.]*6, size=(3, 2)) + desired = np.array([[[4, 3, 5, 4, 2, 2], + [5, 2, 8, 2, 2, 1]], + [[3, 4, 3, 6, 0, 4], + [2, 1, 4, 3, 6, 4]], + [[4, 4, 2, 5, 2, 3], + [4, 3, 4, 2, 3, 4]]]) + assert_array_equal(actual, desired) + + def test_multivariate_normal(self): + np.random.seed(self.seed) + mean = (.123456789, 10) + cov = [[1, 0], [0, 1]] + size = (3, 2) + actual = np.random.multivariate_normal(mean, cov, size) + desired = np.array([[[1.463620246718631, 11.73759122771936], + [1.622445133300628, 9.771356667546383]], + [[2.154490787682787, 12.170324946056553], + [1.719909438201865, 9.230548443648306]], + [[0.689515026297799, 9.880729819607714], + [-0.023054015651998, 9.201096623542879]]]) + + assert_array_almost_equal(actual, desired, decimal=15) + + # Check for default size, was raising deprecation warning + actual = np.random.multivariate_normal(mean, cov) + desired = np.array([0.895289569463708, 9.17180864067987]) + assert_array_almost_equal(actual, desired, decimal=15) + + # Check that non positive-semidefinite covariance warns with + # RuntimeWarning + mean = [0, 0] + cov = [[1, 2], [2, 1]] + assert_warns(RuntimeWarning, np.random.multivariate_normal, mean, cov) + + # and that it doesn't warn with RuntimeWarning check_valid='ignore' + assert_no_warnings(np.random.multivariate_normal, mean, cov, + check_valid='ignore') + + # and that it raises with RuntimeWarning check_valid='raises' + assert_raises(ValueError, np.random.multivariate_normal, mean, cov, + check_valid='raise') + + cov = np.array([[1, 0.1], [0.1, 1]], dtype=np.float32) + with suppress_warnings() as sup: + np.random.multivariate_normal(mean, cov) + w = sup.record(RuntimeWarning) + assert len(w) == 0 + + def test_negative_binomial(self): + np.random.seed(self.seed) + actual = np.random.negative_binomial(n=100, p=.12345, size=(3, 2)) + desired = np.array([[848, 841], + [892, 611], + [779, 647]]) + assert_array_equal(actual, desired) + + def test_noncentral_chisquare(self): + np.random.seed(self.seed) + actual = np.random.noncentral_chisquare(df=5, nonc=5, size=(3, 2)) + desired = np.array([[23.91905354498517511, 13.35324692733826346], + [31.22452661329736401, 16.60047399466177254], + [5.03461598262724586, 17.94973089023519464]]) + assert_array_almost_equal(actual, desired, decimal=14) + + actual = np.random.noncentral_chisquare(df=.5, nonc=.2, size=(3, 2)) + desired = np.array([[1.47145377828516666, 0.15052899268012659], + [0.00943803056963588, 1.02647251615666169], + [0.332334982684171, 0.15451287602753125]]) + assert_array_almost_equal(actual, desired, decimal=14) + + np.random.seed(self.seed) + actual = np.random.noncentral_chisquare(df=5, nonc=0, size=(3, 2)) + desired = np.array([[9.597154162763948, 11.725484450296079], + [10.413711048138335, 3.694475922923986], + [13.484222138963087, 14.377255424602957]]) + assert_array_almost_equal(actual, desired, decimal=14) + + def test_noncentral_f(self): + np.random.seed(self.seed) + actual = np.random.noncentral_f(dfnum=5, dfden=2, nonc=1, + size=(3, 2)) + desired = np.array([[1.40598099674926669, 0.34207973179285761], + [3.57715069265772545, 7.92632662577829805], + [0.43741599463544162, 1.1774208752428319]]) + assert_array_almost_equal(actual, desired, decimal=14) + + def test_normal(self): + np.random.seed(self.seed) + actual = np.random.normal(loc=.123456789, scale=2.0, size=(3, 2)) + desired = np.array([[2.80378370443726244, 3.59863924443872163], + [3.121433477601256, -0.33382987590723379], + [4.18552478636557357, 4.46410668111310471]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_normal_0(self): + assert_equal(np.random.normal(scale=0), 0) + assert_raises(ValueError, np.random.normal, scale=-0.) + + def test_pareto(self): + np.random.seed(self.seed) + actual = np.random.pareto(a=.123456789, size=(3, 2)) + desired = np.array( + [[2.46852460439034849e+03, 1.41286880810518346e+03], + [5.28287797029485181e+07, 6.57720981047328785e+07], + [1.40840323350391515e+02, 1.98390255135251704e+05]]) + # For some reason on 32-bit x86 Ubuntu 12.10 the [1, 0] entry in this + # matrix differs by 24 nulps. Discussion: + # https://mail.python.org/pipermail/numpy-discussion/2012-September/063801.html + # Consensus is that this is probably some gcc quirk that affects + # rounding but not in any important way, so we just use a looser + # tolerance on this test: + np.testing.assert_array_almost_equal_nulp(actual, desired, nulp=30) + + def test_poisson(self): + np.random.seed(self.seed) + actual = np.random.poisson(lam=.123456789, size=(3, 2)) + desired = np.array([[0, 0], + [1, 0], + [0, 0]]) + assert_array_equal(actual, desired) + + def test_poisson_exceptions(self): + lambig = np.iinfo('l').max + lamneg = -1 + assert_raises(ValueError, np.random.poisson, lamneg) + assert_raises(ValueError, np.random.poisson, [lamneg]*10) + assert_raises(ValueError, np.random.poisson, lambig) + assert_raises(ValueError, np.random.poisson, [lambig]*10) + + def test_power(self): + np.random.seed(self.seed) + actual = np.random.power(a=.123456789, size=(3, 2)) + desired = np.array([[0.02048932883240791, 0.01424192241128213], + [0.38446073748535298, 0.39499689943484395], + [0.00177699707563439, 0.13115505880863756]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_rayleigh(self): + np.random.seed(self.seed) + actual = np.random.rayleigh(scale=10, size=(3, 2)) + desired = np.array([[13.8882496494248393, 13.383318339044731], + [20.95413364294492098, 21.08285015800712614], + [11.06066537006854311, 17.35468505778271009]]) + assert_array_almost_equal(actual, desired, decimal=14) + + def test_rayleigh_0(self): + assert_equal(np.random.rayleigh(scale=0), 0) + assert_raises(ValueError, np.random.rayleigh, scale=-0.) + + def test_standard_cauchy(self): + np.random.seed(self.seed) + actual = np.random.standard_cauchy(size=(3, 2)) + desired = np.array([[0.77127660196445336, -6.55601161955910605], + [0.93582023391158309, -2.07479293013759447], + [-4.74601644297011926, 0.18338989290760804]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_standard_exponential(self): + np.random.seed(self.seed) + actual = np.random.standard_exponential(size=(3, 2)) + desired = np.array([[0.96441739162374596, 0.89556604882105506], + [2.1953785836319808, 2.22243285392490542], + [0.6116915921431676, 1.50592546727413201]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_standard_gamma(self): + np.random.seed(self.seed) + actual = np.random.standard_gamma(shape=3, size=(3, 2)) + desired = np.array([[5.50841531318455058, 6.62953470301903103], + [5.93988484943779227, 2.31044849402133989], + [7.54838614231317084, 8.012756093271868]]) + assert_array_almost_equal(actual, desired, decimal=14) + + def test_standard_gamma_0(self): + assert_equal(np.random.standard_gamma(shape=0), 0) + assert_raises(ValueError, np.random.standard_gamma, shape=-0.) + + def test_standard_normal(self): + np.random.seed(self.seed) + actual = np.random.standard_normal(size=(3, 2)) + desired = np.array([[1.34016345771863121, 1.73759122771936081], + [1.498988344300628, -0.2286433324536169], + [2.031033998682787, 2.17032494605655257]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_standard_t(self): + np.random.seed(self.seed) + actual = np.random.standard_t(df=10, size=(3, 2)) + desired = np.array([[0.97140611862659965, -0.08830486548450577], + [1.36311143689505321, -0.55317463909867071], + [-0.18473749069684214, 0.61181537341755321]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_triangular(self): + np.random.seed(self.seed) + actual = np.random.triangular(left=5.12, mode=10.23, right=20.34, + size=(3, 2)) + desired = np.array([[12.68117178949215784, 12.4129206149193152], + [16.20131377335158263, 16.25692138747600524], + [11.20400690911820263, 14.4978144835829923]]) + assert_array_almost_equal(actual, desired, decimal=14) + + def test_uniform(self): + np.random.seed(self.seed) + actual = np.random.uniform(low=1.23, high=10.54, size=(3, 2)) + desired = np.array([[6.99097932346268003, 6.73801597444323974], + [9.50364421400426274, 9.53130618907631089], + [5.48995325769805476, 8.47493103280052118]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_uniform_range_bounds(self): + fmin = np.finfo('float').min + fmax = np.finfo('float').max + + func = np.random.uniform + assert_raises(OverflowError, func, -np.inf, 0) + assert_raises(OverflowError, func, 0, np.inf) + assert_raises(OverflowError, func, fmin, fmax) + assert_raises(OverflowError, func, [-np.inf], [0]) + assert_raises(OverflowError, func, [0], [np.inf]) + + # (fmax / 1e17) - fmin is within range, so this should not throw + # account for i386 extended precision DBL_MAX / 1e17 + DBL_MAX > + # DBL_MAX by increasing fmin a bit + np.random.uniform(low=np.nextafter(fmin, 1), high=fmax / 1e17) + + def test_scalar_exception_propagation(self): + # Tests that exceptions are correctly propagated in distributions + # when called with objects that throw exceptions when converted to + # scalars. + # + # Regression test for gh: 8865 + + class ThrowingFloat(np.ndarray): + def __float__(self): + raise TypeError + + throwing_float = np.array(1.0).view(ThrowingFloat) + assert_raises(TypeError, np.random.uniform, throwing_float, + throwing_float) + + class ThrowingInteger(np.ndarray): + def __int__(self): + raise TypeError + + __index__ = __int__ + + throwing_int = np.array(1).view(ThrowingInteger) + assert_raises(TypeError, np.random.hypergeometric, throwing_int, 1, 1) + + def test_vonmises(self): + np.random.seed(self.seed) + actual = np.random.vonmises(mu=1.23, kappa=1.54, size=(3, 2)) + desired = np.array([[2.28567572673902042, 2.89163838442285037], + [0.38198375564286025, 2.57638023113890746], + [1.19153771588353052, 1.83509849681825354]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_vonmises_small(self): + # check infinite loop, gh-4720 + np.random.seed(self.seed) + r = np.random.vonmises(mu=0., kappa=1.1e-8, size=10**6) + np.testing.assert_(np.isfinite(r).all()) + + def test_wald(self): + np.random.seed(self.seed) + actual = np.random.wald(mean=1.23, scale=1.54, size=(3, 2)) + desired = np.array([[3.82935265715889983, 5.13125249184285526], + [0.35045403618358717, 1.50832396872003538], + [0.24124319895843183, 0.22031101461955038]]) + assert_array_almost_equal(actual, desired, decimal=14) + + def test_weibull(self): + np.random.seed(self.seed) + actual = np.random.weibull(a=1.23, size=(3, 2)) + desired = np.array([[0.97097342648766727, 0.91422896443565516], + [1.89517770034962929, 1.91414357960479564], + [0.67057783752390987, 1.39494046635066793]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_weibull_0(self): + np.random.seed(self.seed) + assert_equal(np.random.weibull(a=0, size=12), np.zeros(12)) + assert_raises(ValueError, np.random.weibull, a=-0.) + + def test_zipf(self): + np.random.seed(self.seed) + actual = np.random.zipf(a=1.23, size=(3, 2)) + desired = np.array([[66, 29], + [1, 1], + [3, 13]]) + assert_array_equal(actual, desired) + + +class TestBroadcast: + # tests that functions that broadcast behave + # correctly when presented with non-scalar arguments + def setup_method(self): + self.seed = 123456789 + + def setSeed(self): + np.random.seed(self.seed) + + # TODO: Include test for randint once it can broadcast + # Can steal the test written in PR #6938 + + def test_uniform(self): + low = [0] + high = [1] + uniform = np.random.uniform + desired = np.array([0.53283302478975902, + 0.53413660089041659, + 0.50955303552646702]) + + self.setSeed() + actual = uniform(low * 3, high) + assert_array_almost_equal(actual, desired, decimal=14) + + self.setSeed() + actual = uniform(low, high * 3) + assert_array_almost_equal(actual, desired, decimal=14) + + def test_normal(self): + loc = [0] + scale = [1] + bad_scale = [-1] + normal = np.random.normal + desired = np.array([2.2129019979039612, + 2.1283977976520019, + 1.8417114045748335]) + + self.setSeed() + actual = normal(loc * 3, scale) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, normal, loc * 3, bad_scale) + + self.setSeed() + actual = normal(loc, scale * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, normal, loc, bad_scale * 3) + + def test_beta(self): + a = [1] + b = [2] + bad_a = [-1] + bad_b = [-2] + beta = np.random.beta + desired = np.array([0.19843558305989056, + 0.075230336409423643, + 0.24976865978980844]) + + self.setSeed() + actual = beta(a * 3, b) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, beta, bad_a * 3, b) + assert_raises(ValueError, beta, a * 3, bad_b) + + self.setSeed() + actual = beta(a, b * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, beta, bad_a, b * 3) + assert_raises(ValueError, beta, a, bad_b * 3) + + def test_exponential(self): + scale = [1] + bad_scale = [-1] + exponential = np.random.exponential + desired = np.array([0.76106853658845242, + 0.76386282278691653, + 0.71243813125891797]) + + self.setSeed() + actual = exponential(scale * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, exponential, bad_scale * 3) + + def test_standard_gamma(self): + shape = [1] + bad_shape = [-1] + std_gamma = np.random.standard_gamma + desired = np.array([0.76106853658845242, + 0.76386282278691653, + 0.71243813125891797]) + + self.setSeed() + actual = std_gamma(shape * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, std_gamma, bad_shape * 3) + + def test_gamma(self): + shape = [1] + scale = [2] + bad_shape = [-1] + bad_scale = [-2] + gamma = np.random.gamma + desired = np.array([1.5221370731769048, + 1.5277256455738331, + 1.4248762625178359]) + + self.setSeed() + actual = gamma(shape * 3, scale) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, gamma, bad_shape * 3, scale) + assert_raises(ValueError, gamma, shape * 3, bad_scale) + + self.setSeed() + actual = gamma(shape, scale * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, gamma, bad_shape, scale * 3) + assert_raises(ValueError, gamma, shape, bad_scale * 3) + + def test_f(self): + dfnum = [1] + dfden = [2] + bad_dfnum = [-1] + bad_dfden = [-2] + f = np.random.f + desired = np.array([0.80038951638264799, + 0.86768719635363512, + 2.7251095168386801]) + + self.setSeed() + actual = f(dfnum * 3, dfden) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, f, bad_dfnum * 3, dfden) + assert_raises(ValueError, f, dfnum * 3, bad_dfden) + + self.setSeed() + actual = f(dfnum, dfden * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, f, bad_dfnum, dfden * 3) + assert_raises(ValueError, f, dfnum, bad_dfden * 3) + + def test_noncentral_f(self): + dfnum = [2] + dfden = [3] + nonc = [4] + bad_dfnum = [0] + bad_dfden = [-1] + bad_nonc = [-2] + nonc_f = np.random.noncentral_f + desired = np.array([9.1393943263705211, + 13.025456344595602, + 8.8018098359100545]) + + self.setSeed() + actual = nonc_f(dfnum * 3, dfden, nonc) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, nonc_f, bad_dfnum * 3, dfden, nonc) + assert_raises(ValueError, nonc_f, dfnum * 3, bad_dfden, nonc) + assert_raises(ValueError, nonc_f, dfnum * 3, dfden, bad_nonc) + + self.setSeed() + actual = nonc_f(dfnum, dfden * 3, nonc) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, nonc_f, bad_dfnum, dfden * 3, nonc) + assert_raises(ValueError, nonc_f, dfnum, bad_dfden * 3, nonc) + assert_raises(ValueError, nonc_f, dfnum, dfden * 3, bad_nonc) + + self.setSeed() + actual = nonc_f(dfnum, dfden, nonc * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, nonc_f, bad_dfnum, dfden, nonc * 3) + assert_raises(ValueError, nonc_f, dfnum, bad_dfden, nonc * 3) + assert_raises(ValueError, nonc_f, dfnum, dfden, bad_nonc * 3) + + def test_noncentral_f_small_df(self): + self.setSeed() + desired = np.array([6.869638627492048, 0.785880199263955]) + actual = np.random.noncentral_f(0.9, 0.9, 2, size=2) + assert_array_almost_equal(actual, desired, decimal=14) + + def test_chisquare(self): + df = [1] + bad_df = [-1] + chisquare = np.random.chisquare + desired = np.array([0.57022801133088286, + 0.51947702108840776, + 0.1320969254923558]) + + self.setSeed() + actual = chisquare(df * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, chisquare, bad_df * 3) + + def test_noncentral_chisquare(self): + df = [1] + nonc = [2] + bad_df = [-1] + bad_nonc = [-2] + nonc_chi = np.random.noncentral_chisquare + desired = np.array([9.0015599467913763, + 4.5804135049718742, + 6.0872302432834564]) + + self.setSeed() + actual = nonc_chi(df * 3, nonc) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, nonc_chi, bad_df * 3, nonc) + assert_raises(ValueError, nonc_chi, df * 3, bad_nonc) + + self.setSeed() + actual = nonc_chi(df, nonc * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, nonc_chi, bad_df, nonc * 3) + assert_raises(ValueError, nonc_chi, df, bad_nonc * 3) + + def test_standard_t(self): + df = [1] + bad_df = [-1] + t = np.random.standard_t + desired = np.array([3.0702872575217643, + 5.8560725167361607, + 1.0274791436474273]) + + self.setSeed() + actual = t(df * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, t, bad_df * 3) + + def test_vonmises(self): + mu = [2] + kappa = [1] + bad_kappa = [-1] + vonmises = np.random.vonmises + desired = np.array([2.9883443664201312, + -2.7064099483995943, + -1.8672476700665914]) + + self.setSeed() + actual = vonmises(mu * 3, kappa) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, vonmises, mu * 3, bad_kappa) + + self.setSeed() + actual = vonmises(mu, kappa * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, vonmises, mu, bad_kappa * 3) + + def test_pareto(self): + a = [1] + bad_a = [-1] + pareto = np.random.pareto + desired = np.array([1.1405622680198362, + 1.1465519762044529, + 1.0389564467453547]) + + self.setSeed() + actual = pareto(a * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, pareto, bad_a * 3) + + def test_weibull(self): + a = [1] + bad_a = [-1] + weibull = np.random.weibull + desired = np.array([0.76106853658845242, + 0.76386282278691653, + 0.71243813125891797]) + + self.setSeed() + actual = weibull(a * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, weibull, bad_a * 3) + + def test_power(self): + a = [1] + bad_a = [-1] + power = np.random.power + desired = np.array([0.53283302478975902, + 0.53413660089041659, + 0.50955303552646702]) + + self.setSeed() + actual = power(a * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, power, bad_a * 3) + + def test_laplace(self): + loc = [0] + scale = [1] + bad_scale = [-1] + laplace = np.random.laplace + desired = np.array([0.067921356028507157, + 0.070715642226971326, + 0.019290950698972624]) + + self.setSeed() + actual = laplace(loc * 3, scale) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, laplace, loc * 3, bad_scale) + + self.setSeed() + actual = laplace(loc, scale * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, laplace, loc, bad_scale * 3) + + def test_gumbel(self): + loc = [0] + scale = [1] + bad_scale = [-1] + gumbel = np.random.gumbel + desired = np.array([0.2730318639556768, + 0.26936705726291116, + 0.33906220393037939]) + + self.setSeed() + actual = gumbel(loc * 3, scale) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, gumbel, loc * 3, bad_scale) + + self.setSeed() + actual = gumbel(loc, scale * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, gumbel, loc, bad_scale * 3) + + def test_logistic(self): + loc = [0] + scale = [1] + bad_scale = [-1] + logistic = np.random.logistic + desired = np.array([0.13152135837586171, + 0.13675915696285773, + 0.038216792802833396]) + + self.setSeed() + actual = logistic(loc * 3, scale) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, logistic, loc * 3, bad_scale) + + self.setSeed() + actual = logistic(loc, scale * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, logistic, loc, bad_scale * 3) + + def test_lognormal(self): + mean = [0] + sigma = [1] + bad_sigma = [-1] + lognormal = np.random.lognormal + desired = np.array([9.1422086044848427, + 8.4013952870126261, + 6.3073234116578671]) + + self.setSeed() + actual = lognormal(mean * 3, sigma) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, lognormal, mean * 3, bad_sigma) + + self.setSeed() + actual = lognormal(mean, sigma * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, lognormal, mean, bad_sigma * 3) + + def test_rayleigh(self): + scale = [1] + bad_scale = [-1] + rayleigh = np.random.rayleigh + desired = np.array([1.2337491937897689, + 1.2360119924878694, + 1.1936818095781789]) + + self.setSeed() + actual = rayleigh(scale * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, rayleigh, bad_scale * 3) + + def test_wald(self): + mean = [0.5] + scale = [1] + bad_mean = [0] + bad_scale = [-2] + wald = np.random.wald + desired = np.array([0.11873681120271318, + 0.12450084820795027, + 0.9096122728408238]) + + self.setSeed() + actual = wald(mean * 3, scale) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, wald, bad_mean * 3, scale) + assert_raises(ValueError, wald, mean * 3, bad_scale) + + self.setSeed() + actual = wald(mean, scale * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, wald, bad_mean, scale * 3) + assert_raises(ValueError, wald, mean, bad_scale * 3) + assert_raises(ValueError, wald, 0.0, 1) + assert_raises(ValueError, wald, 0.5, 0.0) + + def test_triangular(self): + left = [1] + right = [3] + mode = [2] + bad_left_one = [3] + bad_mode_one = [4] + bad_left_two, bad_mode_two = right * 2 + triangular = np.random.triangular + desired = np.array([2.03339048710429, + 2.0347400359389356, + 2.0095991069536208]) + + self.setSeed() + actual = triangular(left * 3, mode, right) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, triangular, bad_left_one * 3, mode, right) + assert_raises(ValueError, triangular, left * 3, bad_mode_one, right) + assert_raises(ValueError, triangular, bad_left_two * 3, bad_mode_two, + right) + + self.setSeed() + actual = triangular(left, mode * 3, right) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, triangular, bad_left_one, mode * 3, right) + assert_raises(ValueError, triangular, left, bad_mode_one * 3, right) + assert_raises(ValueError, triangular, bad_left_two, bad_mode_two * 3, + right) + + self.setSeed() + actual = triangular(left, mode, right * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, triangular, bad_left_one, mode, right * 3) + assert_raises(ValueError, triangular, left, bad_mode_one, right * 3) + assert_raises(ValueError, triangular, bad_left_two, bad_mode_two, + right * 3) + + def test_binomial(self): + n = [1] + p = [0.5] + bad_n = [-1] + bad_p_one = [-1] + bad_p_two = [1.5] + binom = np.random.binomial + desired = np.array([1, 1, 1]) + + self.setSeed() + actual = binom(n * 3, p) + assert_array_equal(actual, desired) + assert_raises(ValueError, binom, bad_n * 3, p) + assert_raises(ValueError, binom, n * 3, bad_p_one) + assert_raises(ValueError, binom, n * 3, bad_p_two) + + self.setSeed() + actual = binom(n, p * 3) + assert_array_equal(actual, desired) + assert_raises(ValueError, binom, bad_n, p * 3) + assert_raises(ValueError, binom, n, bad_p_one * 3) + assert_raises(ValueError, binom, n, bad_p_two * 3) + + def test_negative_binomial(self): + n = [1] + p = [0.5] + bad_n = [-1] + bad_p_one = [-1] + bad_p_two = [1.5] + neg_binom = np.random.negative_binomial + desired = np.array([1, 0, 1]) + + self.setSeed() + actual = neg_binom(n * 3, p) + assert_array_equal(actual, desired) + assert_raises(ValueError, neg_binom, bad_n * 3, p) + assert_raises(ValueError, neg_binom, n * 3, bad_p_one) + assert_raises(ValueError, neg_binom, n * 3, bad_p_two) + + self.setSeed() + actual = neg_binom(n, p * 3) + assert_array_equal(actual, desired) + assert_raises(ValueError, neg_binom, bad_n, p * 3) + assert_raises(ValueError, neg_binom, n, bad_p_one * 3) + assert_raises(ValueError, neg_binom, n, bad_p_two * 3) + + def test_poisson(self): + max_lam = np.random.RandomState()._poisson_lam_max + + lam = [1] + bad_lam_one = [-1] + bad_lam_two = [max_lam * 2] + poisson = np.random.poisson + desired = np.array([1, 1, 0]) + + self.setSeed() + actual = poisson(lam * 3) + assert_array_equal(actual, desired) + assert_raises(ValueError, poisson, bad_lam_one * 3) + assert_raises(ValueError, poisson, bad_lam_two * 3) + + def test_zipf(self): + a = [2] + bad_a = [0] + zipf = np.random.zipf + desired = np.array([2, 2, 1]) + + self.setSeed() + actual = zipf(a * 3) + assert_array_equal(actual, desired) + assert_raises(ValueError, zipf, bad_a * 3) + with np.errstate(invalid='ignore'): + assert_raises(ValueError, zipf, np.nan) + assert_raises(ValueError, zipf, [0, 0, np.nan]) + + def test_geometric(self): + p = [0.5] + bad_p_one = [-1] + bad_p_two = [1.5] + geom = np.random.geometric + desired = np.array([2, 2, 2]) + + self.setSeed() + actual = geom(p * 3) + assert_array_equal(actual, desired) + assert_raises(ValueError, geom, bad_p_one * 3) + assert_raises(ValueError, geom, bad_p_two * 3) + + def test_hypergeometric(self): + ngood = [1] + nbad = [2] + nsample = [2] + bad_ngood = [-1] + bad_nbad = [-2] + bad_nsample_one = [0] + bad_nsample_two = [4] + hypergeom = np.random.hypergeometric + desired = np.array([1, 1, 1]) + + self.setSeed() + actual = hypergeom(ngood * 3, nbad, nsample) + assert_array_equal(actual, desired) + assert_raises(ValueError, hypergeom, bad_ngood * 3, nbad, nsample) + assert_raises(ValueError, hypergeom, ngood * 3, bad_nbad, nsample) + assert_raises(ValueError, hypergeom, ngood * 3, nbad, bad_nsample_one) + assert_raises(ValueError, hypergeom, ngood * 3, nbad, bad_nsample_two) + + self.setSeed() + actual = hypergeom(ngood, nbad * 3, nsample) + assert_array_equal(actual, desired) + assert_raises(ValueError, hypergeom, bad_ngood, nbad * 3, nsample) + assert_raises(ValueError, hypergeom, ngood, bad_nbad * 3, nsample) + assert_raises(ValueError, hypergeom, ngood, nbad * 3, bad_nsample_one) + assert_raises(ValueError, hypergeom, ngood, nbad * 3, bad_nsample_two) + + self.setSeed() + actual = hypergeom(ngood, nbad, nsample * 3) + assert_array_equal(actual, desired) + assert_raises(ValueError, hypergeom, bad_ngood, nbad, nsample * 3) + assert_raises(ValueError, hypergeom, ngood, bad_nbad, nsample * 3) + assert_raises(ValueError, hypergeom, ngood, nbad, bad_nsample_one * 3) + assert_raises(ValueError, hypergeom, ngood, nbad, bad_nsample_two * 3) + + def test_logseries(self): + p = [0.5] + bad_p_one = [2] + bad_p_two = [-1] + logseries = np.random.logseries + desired = np.array([1, 1, 1]) + + self.setSeed() + actual = logseries(p * 3) + assert_array_equal(actual, desired) + assert_raises(ValueError, logseries, bad_p_one * 3) + assert_raises(ValueError, logseries, bad_p_two * 3) + + +@pytest.mark.skipif(IS_WASM, reason="can't start thread") +class TestThread: + # make sure each state produces the same sequence even in threads + def setup_method(self): + self.seeds = range(4) + + def check_function(self, function, sz): + from threading import Thread + + out1 = np.empty((len(self.seeds),) + sz) + out2 = np.empty((len(self.seeds),) + sz) + + # threaded generation + t = [Thread(target=function, args=(np.random.RandomState(s), o)) + for s, o in zip(self.seeds, out1)] + [x.start() for x in t] + [x.join() for x in t] + + # the same serial + for s, o in zip(self.seeds, out2): + function(np.random.RandomState(s), o) + + # these platforms change x87 fpu precision mode in threads + if np.intp().dtype.itemsize == 4 and sys.platform == "win32": + assert_array_almost_equal(out1, out2) + else: + assert_array_equal(out1, out2) + + def test_normal(self): + def gen_random(state, out): + out[...] = state.normal(size=10000) + self.check_function(gen_random, sz=(10000,)) + + def test_exp(self): + def gen_random(state, out): + out[...] = state.exponential(scale=np.ones((100, 1000))) + self.check_function(gen_random, sz=(100, 1000)) + + def test_multinomial(self): + def gen_random(state, out): + out[...] = state.multinomial(10, [1/6.]*6, size=10000) + self.check_function(gen_random, sz=(10000, 6)) + + +# See Issue #4263 +class TestSingleEltArrayInput: + def setup_method(self): + self.argOne = np.array([2]) + self.argTwo = np.array([3]) + self.argThree = np.array([4]) + self.tgtShape = (1,) + + def test_one_arg_funcs(self): + funcs = (np.random.exponential, np.random.standard_gamma, + np.random.chisquare, np.random.standard_t, + np.random.pareto, np.random.weibull, + np.random.power, np.random.rayleigh, + np.random.poisson, np.random.zipf, + np.random.geometric, np.random.logseries) + + probfuncs = (np.random.geometric, np.random.logseries) + + for func in funcs: + if func in probfuncs: # p < 1.0 + out = func(np.array([0.5])) + + else: + out = func(self.argOne) + + assert_equal(out.shape, self.tgtShape) + + def test_two_arg_funcs(self): + funcs = (np.random.uniform, np.random.normal, + np.random.beta, np.random.gamma, + np.random.f, np.random.noncentral_chisquare, + np.random.vonmises, np.random.laplace, + np.random.gumbel, np.random.logistic, + np.random.lognormal, np.random.wald, + np.random.binomial, np.random.negative_binomial) + + probfuncs = (np.random.binomial, np.random.negative_binomial) + + for func in funcs: + if func in probfuncs: # p <= 1 + argTwo = np.array([0.5]) + + else: + argTwo = self.argTwo + + out = func(self.argOne, argTwo) + assert_equal(out.shape, self.tgtShape) + + out = func(self.argOne[0], argTwo) + assert_equal(out.shape, self.tgtShape) + + out = func(self.argOne, argTwo[0]) + assert_equal(out.shape, self.tgtShape) + + def test_randint(self): + itype = [bool, np.int8, np.uint8, np.int16, np.uint16, + np.int32, np.uint32, np.int64, np.uint64] + func = np.random.randint + high = np.array([1]) + low = np.array([0]) + + for dt in itype: + out = func(low, high, dtype=dt) + assert_equal(out.shape, self.tgtShape) + + out = func(low[0], high, dtype=dt) + assert_equal(out.shape, self.tgtShape) + + out = func(low, high[0], dtype=dt) + assert_equal(out.shape, self.tgtShape) + + def test_three_arg_funcs(self): + funcs = [np.random.noncentral_f, np.random.triangular, + np.random.hypergeometric] + + for func in funcs: + out = func(self.argOne, self.argTwo, self.argThree) + assert_equal(out.shape, self.tgtShape) + + out = func(self.argOne[0], self.argTwo, self.argThree) + assert_equal(out.shape, self.tgtShape) + + out = func(self.argOne, self.argTwo[0], self.argThree) + assert_equal(out.shape, self.tgtShape) diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/test_randomstate.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/test_randomstate.py new file mode 100644 index 0000000000000000000000000000000000000000..c77bfce883aea276304c817be9ef93584b59cb28 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/test_randomstate.py @@ -0,0 +1,2121 @@ +import hashlib +import pickle +import sys +import warnings + +import numpy as np +import pytest +from numpy.testing import ( + assert_, assert_raises, assert_equal, assert_warns, + assert_no_warnings, assert_array_equal, assert_array_almost_equal, + suppress_warnings, IS_WASM + ) + +from numpy.random import MT19937, PCG64 +from numpy import random + +INT_FUNCS = {'binomial': (100.0, 0.6), + 'geometric': (.5,), + 'hypergeometric': (20, 20, 10), + 'logseries': (.5,), + 'multinomial': (20, np.ones(6) / 6.0), + 'negative_binomial': (100, .5), + 'poisson': (10.0,), + 'zipf': (2,), + } + +if np.iinfo(int).max < 2**32: + # Windows and some 32-bit platforms, e.g., ARM + INT_FUNC_HASHES = {'binomial': '2fbead005fc63942decb5326d36a1f32fe2c9d32c904ee61e46866b88447c263', + 'logseries': '23ead5dcde35d4cfd4ef2c105e4c3d43304b45dc1b1444b7823b9ee4fa144ebb', + 'geometric': '0d764db64f5c3bad48c8c33551c13b4d07a1e7b470f77629bef6c985cac76fcf', + 'hypergeometric': '7b59bf2f1691626c5815cdcd9a49e1dd68697251d4521575219e4d2a1b8b2c67', + 'multinomial': 'd754fa5b92943a38ec07630de92362dd2e02c43577fc147417dc5b9db94ccdd3', + 'negative_binomial': '8eb216f7cb2a63cf55605422845caaff002fddc64a7dc8b2d45acd477a49e824', + 'poisson': '70c891d76104013ebd6f6bcf30d403a9074b886ff62e4e6b8eb605bf1a4673b7', + 'zipf': '01f074f97517cd5d21747148ac6ca4074dde7fcb7acbaec0a936606fecacd93f', + } +else: + INT_FUNC_HASHES = {'binomial': '8626dd9d052cb608e93d8868de0a7b347258b199493871a1dc56e2a26cacb112', + 'geometric': '8edd53d272e49c4fc8fbbe6c7d08d563d62e482921f3131d0a0e068af30f0db9', + 'hypergeometric': '83496cc4281c77b786c9b7ad88b74d42e01603a55c60577ebab81c3ba8d45657', + 'logseries': '65878a38747c176bc00e930ebafebb69d4e1e16cd3a704e264ea8f5e24f548db', + 'multinomial': '7a984ae6dca26fd25374479e118b22f55db0aedccd5a0f2584ceada33db98605', + 'negative_binomial': 'd636d968e6a24ae92ab52fe11c46ac45b0897e98714426764e820a7d77602a61', + 'poisson': '956552176f77e7c9cb20d0118fc9cf690be488d790ed4b4c4747b965e61b0bb4', + 'zipf': 'f84ba7feffda41e606e20b28dfc0f1ea9964a74574513d4a4cbc98433a8bfa45', + } + + +@pytest.fixture(scope='module', params=INT_FUNCS) +def int_func(request): + return (request.param, INT_FUNCS[request.param], + INT_FUNC_HASHES[request.param]) + + +@pytest.fixture +def restore_singleton_bitgen(): + """Ensures that the singleton bitgen is restored after a test""" + orig_bitgen = np.random.get_bit_generator() + yield + np.random.set_bit_generator(orig_bitgen) + + +def assert_mt19937_state_equal(a, b): + assert_equal(a['bit_generator'], b['bit_generator']) + assert_array_equal(a['state']['key'], b['state']['key']) + assert_array_equal(a['state']['pos'], b['state']['pos']) + assert_equal(a['has_gauss'], b['has_gauss']) + assert_equal(a['gauss'], b['gauss']) + + +class TestSeed: + def test_scalar(self): + s = random.RandomState(0) + assert_equal(s.randint(1000), 684) + s = random.RandomState(4294967295) + assert_equal(s.randint(1000), 419) + + def test_array(self): + s = random.RandomState(range(10)) + assert_equal(s.randint(1000), 468) + s = random.RandomState(np.arange(10)) + assert_equal(s.randint(1000), 468) + s = random.RandomState([0]) + assert_equal(s.randint(1000), 973) + s = random.RandomState([4294967295]) + assert_equal(s.randint(1000), 265) + + def test_invalid_scalar(self): + # seed must be an unsigned 32 bit integer + assert_raises(TypeError, random.RandomState, -0.5) + assert_raises(ValueError, random.RandomState, -1) + + def test_invalid_array(self): + # seed must be an unsigned 32 bit integer + assert_raises(TypeError, random.RandomState, [-0.5]) + assert_raises(ValueError, random.RandomState, [-1]) + assert_raises(ValueError, random.RandomState, [4294967296]) + assert_raises(ValueError, random.RandomState, [1, 2, 4294967296]) + assert_raises(ValueError, random.RandomState, [1, -2, 4294967296]) + + def test_invalid_array_shape(self): + # gh-9832 + assert_raises(ValueError, random.RandomState, np.array([], + dtype=np.int64)) + assert_raises(ValueError, random.RandomState, [[1, 2, 3]]) + assert_raises(ValueError, random.RandomState, [[1, 2, 3], + [4, 5, 6]]) + + def test_cannot_seed(self): + rs = random.RandomState(PCG64(0)) + with assert_raises(TypeError): + rs.seed(1234) + + def test_invalid_initialization(self): + assert_raises(ValueError, random.RandomState, MT19937) + + +class TestBinomial: + def test_n_zero(self): + # Tests the corner case of n == 0 for the binomial distribution. + # binomial(0, p) should be zero for any p in [0, 1]. + # This test addresses issue #3480. + zeros = np.zeros(2, dtype='int') + for p in [0, .5, 1]: + assert_(random.binomial(0, p) == 0) + assert_array_equal(random.binomial(zeros, p), zeros) + + def test_p_is_nan(self): + # Issue #4571. + assert_raises(ValueError, random.binomial, 1, np.nan) + + +class TestMultinomial: + def test_basic(self): + random.multinomial(100, [0.2, 0.8]) + + def test_zero_probability(self): + random.multinomial(100, [0.2, 0.8, 0.0, 0.0, 0.0]) + + def test_int_negative_interval(self): + assert_(-5 <= random.randint(-5, -1) < -1) + x = random.randint(-5, -1, 5) + assert_(np.all(-5 <= x)) + assert_(np.all(x < -1)) + + def test_size(self): + # gh-3173 + p = [0.5, 0.5] + assert_equal(random.multinomial(1, p, np.uint32(1)).shape, (1, 2)) + assert_equal(random.multinomial(1, p, np.uint32(1)).shape, (1, 2)) + assert_equal(random.multinomial(1, p, np.uint32(1)).shape, (1, 2)) + assert_equal(random.multinomial(1, p, [2, 2]).shape, (2, 2, 2)) + assert_equal(random.multinomial(1, p, (2, 2)).shape, (2, 2, 2)) + assert_equal(random.multinomial(1, p, np.array((2, 2))).shape, + (2, 2, 2)) + + assert_raises(TypeError, random.multinomial, 1, p, + float(1)) + + def test_invalid_prob(self): + assert_raises(ValueError, random.multinomial, 100, [1.1, 0.2]) + assert_raises(ValueError, random.multinomial, 100, [-.1, 0.9]) + + def test_invalid_n(self): + assert_raises(ValueError, random.multinomial, -1, [0.8, 0.2]) + + def test_p_non_contiguous(self): + p = np.arange(15.) + p /= np.sum(p[1::3]) + pvals = p[1::3] + random.seed(1432985819) + non_contig = random.multinomial(100, pvals=pvals) + random.seed(1432985819) + contig = random.multinomial(100, pvals=np.ascontiguousarray(pvals)) + assert_array_equal(non_contig, contig) + + def test_multinomial_pvals_float32(self): + x = np.array([9.9e-01, 9.9e-01, 1.0e-09, 1.0e-09, 1.0e-09, 1.0e-09, + 1.0e-09, 1.0e-09, 1.0e-09, 1.0e-09], dtype=np.float32) + pvals = x / x.sum() + match = r"[\w\s]*pvals array is cast to 64-bit floating" + with pytest.raises(ValueError, match=match): + random.multinomial(1, pvals) + + def test_multinomial_n_float(self): + # Non-index integer types should gracefully truncate floats + random.multinomial(100.5, [0.2, 0.8]) + +class TestSetState: + def setup_method(self): + self.seed = 1234567890 + self.random_state = random.RandomState(self.seed) + self.state = self.random_state.get_state() + + def test_basic(self): + old = self.random_state.tomaxint(16) + self.random_state.set_state(self.state) + new = self.random_state.tomaxint(16) + assert_(np.all(old == new)) + + def test_gaussian_reset(self): + # Make sure the cached every-other-Gaussian is reset. + old = self.random_state.standard_normal(size=3) + self.random_state.set_state(self.state) + new = self.random_state.standard_normal(size=3) + assert_(np.all(old == new)) + + def test_gaussian_reset_in_media_res(self): + # When the state is saved with a cached Gaussian, make sure the + # cached Gaussian is restored. + + self.random_state.standard_normal() + state = self.random_state.get_state() + old = self.random_state.standard_normal(size=3) + self.random_state.set_state(state) + new = self.random_state.standard_normal(size=3) + assert_(np.all(old == new)) + + def test_backwards_compatibility(self): + # Make sure we can accept old state tuples that do not have the + # cached Gaussian value. + old_state = self.state[:-2] + x1 = self.random_state.standard_normal(size=16) + self.random_state.set_state(old_state) + x2 = self.random_state.standard_normal(size=16) + self.random_state.set_state(self.state) + x3 = self.random_state.standard_normal(size=16) + assert_(np.all(x1 == x2)) + assert_(np.all(x1 == x3)) + + def test_negative_binomial(self): + # Ensure that the negative binomial results take floating point + # arguments without truncation. + self.random_state.negative_binomial(0.5, 0.5) + + def test_get_state_warning(self): + rs = random.RandomState(PCG64()) + with suppress_warnings() as sup: + w = sup.record(RuntimeWarning) + state = rs.get_state() + assert_(len(w) == 1) + assert isinstance(state, dict) + assert state['bit_generator'] == 'PCG64' + + def test_invalid_legacy_state_setting(self): + state = self.random_state.get_state() + new_state = ('Unknown', ) + state[1:] + assert_raises(ValueError, self.random_state.set_state, new_state) + assert_raises(TypeError, self.random_state.set_state, + np.array(new_state, dtype=object)) + state = self.random_state.get_state(legacy=False) + del state['bit_generator'] + assert_raises(ValueError, self.random_state.set_state, state) + + def test_pickle(self): + self.random_state.seed(0) + self.random_state.random_sample(100) + self.random_state.standard_normal() + pickled = self.random_state.get_state(legacy=False) + assert_equal(pickled['has_gauss'], 1) + rs_unpick = pickle.loads(pickle.dumps(self.random_state)) + unpickled = rs_unpick.get_state(legacy=False) + assert_mt19937_state_equal(pickled, unpickled) + + def test_state_setting(self): + attr_state = self.random_state.__getstate__() + self.random_state.standard_normal() + self.random_state.__setstate__(attr_state) + state = self.random_state.get_state(legacy=False) + assert_mt19937_state_equal(attr_state, state) + + def test_repr(self): + assert repr(self.random_state).startswith('RandomState(MT19937)') + + +class TestRandint: + + rfunc = random.randint + + # valid integer/boolean types + itype = [np.bool_, np.int8, np.uint8, np.int16, np.uint16, + np.int32, np.uint32, np.int64, np.uint64] + + def test_unsupported_type(self): + assert_raises(TypeError, self.rfunc, 1, dtype=float) + + def test_bounds_checking(self): + for dt in self.itype: + lbnd = 0 if dt is np.bool_ else np.iinfo(dt).min + ubnd = 2 if dt is np.bool_ else np.iinfo(dt).max + 1 + assert_raises(ValueError, self.rfunc, lbnd - 1, ubnd, dtype=dt) + assert_raises(ValueError, self.rfunc, lbnd, ubnd + 1, dtype=dt) + assert_raises(ValueError, self.rfunc, ubnd, lbnd, dtype=dt) + assert_raises(ValueError, self.rfunc, 1, 0, dtype=dt) + + def test_rng_zero_and_extremes(self): + for dt in self.itype: + lbnd = 0 if dt is np.bool_ else np.iinfo(dt).min + ubnd = 2 if dt is np.bool_ else np.iinfo(dt).max + 1 + + tgt = ubnd - 1 + assert_equal(self.rfunc(tgt, tgt + 1, size=1000, dtype=dt), tgt) + + tgt = lbnd + assert_equal(self.rfunc(tgt, tgt + 1, size=1000, dtype=dt), tgt) + + tgt = (lbnd + ubnd)//2 + assert_equal(self.rfunc(tgt, tgt + 1, size=1000, dtype=dt), tgt) + + def test_full_range(self): + # Test for ticket #1690 + + for dt in self.itype: + lbnd = 0 if dt is np.bool_ else np.iinfo(dt).min + ubnd = 2 if dt is np.bool_ else np.iinfo(dt).max + 1 + + try: + self.rfunc(lbnd, ubnd, dtype=dt) + except Exception as e: + raise AssertionError("No error should have been raised, " + "but one was with the following " + "message:\n\n%s" % str(e)) + + def test_in_bounds_fuzz(self): + # Don't use fixed seed + random.seed() + + for dt in self.itype[1:]: + for ubnd in [4, 8, 16]: + vals = self.rfunc(2, ubnd, size=2**16, dtype=dt) + assert_(vals.max() < ubnd) + assert_(vals.min() >= 2) + + vals = self.rfunc(0, 2, size=2**16, dtype=np.bool_) + + assert_(vals.max() < 2) + assert_(vals.min() >= 0) + + def test_repeatability(self): + # We use a sha256 hash of generated sequences of 1000 samples + # in the range [0, 6) for all but bool, where the range + # is [0, 2). Hashes are for little endian numbers. + tgt = {'bool': '509aea74d792fb931784c4b0135392c65aec64beee12b0cc167548a2c3d31e71', + 'int16': '7b07f1a920e46f6d0fe02314155a2330bcfd7635e708da50e536c5ebb631a7d4', + 'int32': 'e577bfed6c935de944424667e3da285012e741892dcb7051a8f1ce68ab05c92f', + 'int64': '0fbead0b06759df2cfb55e43148822d4a1ff953c7eb19a5b08445a63bb64fa9e', + 'int8': '001aac3a5acb935a9b186cbe14a1ca064b8bb2dd0b045d48abeacf74d0203404', + 'uint16': '7b07f1a920e46f6d0fe02314155a2330bcfd7635e708da50e536c5ebb631a7d4', + 'uint32': 'e577bfed6c935de944424667e3da285012e741892dcb7051a8f1ce68ab05c92f', + 'uint64': '0fbead0b06759df2cfb55e43148822d4a1ff953c7eb19a5b08445a63bb64fa9e', + 'uint8': '001aac3a5acb935a9b186cbe14a1ca064b8bb2dd0b045d48abeacf74d0203404'} + + for dt in self.itype[1:]: + random.seed(1234) + + # view as little endian for hash + if sys.byteorder == 'little': + val = self.rfunc(0, 6, size=1000, dtype=dt) + else: + val = self.rfunc(0, 6, size=1000, dtype=dt).byteswap() + + res = hashlib.sha256(val.view(np.int8)).hexdigest() + assert_(tgt[np.dtype(dt).name] == res) + + # bools do not depend on endianness + random.seed(1234) + val = self.rfunc(0, 2, size=1000, dtype=bool).view(np.int8) + res = hashlib.sha256(val).hexdigest() + assert_(tgt[np.dtype(bool).name] == res) + + @pytest.mark.skipif(np.iinfo('l').max < 2**32, + reason='Cannot test with 32-bit C long') + def test_repeatability_32bit_boundary_broadcasting(self): + desired = np.array([[[3992670689, 2438360420, 2557845020], + [4107320065, 4142558326, 3216529513], + [1605979228, 2807061240, 665605495]], + [[3211410639, 4128781000, 457175120], + [1712592594, 1282922662, 3081439808], + [3997822960, 2008322436, 1563495165]], + [[1398375547, 4269260146, 115316740], + [3414372578, 3437564012, 2112038651], + [3572980305, 2260248732, 3908238631]], + [[2561372503, 223155946, 3127879445], + [ 441282060, 3514786552, 2148440361], + [1629275283, 3479737011, 3003195987]], + [[ 412181688, 940383289, 3047321305], + [2978368172, 764731833, 2282559898], + [ 105711276, 720447391, 3596512484]]]) + for size in [None, (5, 3, 3)]: + random.seed(12345) + x = self.rfunc([[-1], [0], [1]], [2**32 - 1, 2**32, 2**32 + 1], + size=size) + assert_array_equal(x, desired if size is not None else desired[0]) + + def test_int64_uint64_corner_case(self): + # When stored in Numpy arrays, `lbnd` is casted + # as np.int64, and `ubnd` is casted as np.uint64. + # Checking whether `lbnd` >= `ubnd` used to be + # done solely via direct comparison, which is incorrect + # because when Numpy tries to compare both numbers, + # it casts both to np.float64 because there is + # no integer superset of np.int64 and np.uint64. However, + # `ubnd` is too large to be represented in np.float64, + # causing it be round down to np.iinfo(np.int64).max, + # leading to a ValueError because `lbnd` now equals + # the new `ubnd`. + + dt = np.int64 + tgt = np.iinfo(np.int64).max + lbnd = np.int64(np.iinfo(np.int64).max) + ubnd = np.uint64(np.iinfo(np.int64).max + 1) + + # None of these function calls should + # generate a ValueError now. + actual = random.randint(lbnd, ubnd, dtype=dt) + assert_equal(actual, tgt) + + def test_respect_dtype_singleton(self): + # See gh-7203 + for dt in self.itype: + lbnd = 0 if dt is np.bool_ else np.iinfo(dt).min + ubnd = 2 if dt is np.bool_ else np.iinfo(dt).max + 1 + + sample = self.rfunc(lbnd, ubnd, dtype=dt) + assert_equal(sample.dtype, np.dtype(dt)) + + for dt in (bool, int): + lbnd = 0 if dt is bool else np.iinfo(dt).min + ubnd = 2 if dt is bool else np.iinfo(dt).max + 1 + + # gh-7284: Ensure that we get Python data types + sample = self.rfunc(lbnd, ubnd, dtype=dt) + assert_(not hasattr(sample, 'dtype')) + assert_equal(type(sample), dt) + + +class TestRandomDist: + # Make sure the random distribution returns the correct value for a + # given seed + + def setup_method(self): + self.seed = 1234567890 + + def test_rand(self): + random.seed(self.seed) + actual = random.rand(3, 2) + desired = np.array([[0.61879477158567997, 0.59162362775974664], + [0.88868358904449662, 0.89165480011560816], + [0.4575674820298663, 0.7781880808593471]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_rand_singleton(self): + random.seed(self.seed) + actual = random.rand() + desired = 0.61879477158567997 + assert_array_almost_equal(actual, desired, decimal=15) + + def test_randn(self): + random.seed(self.seed) + actual = random.randn(3, 2) + desired = np.array([[1.34016345771863121, 1.73759122771936081], + [1.498988344300628, -0.2286433324536169], + [2.031033998682787, 2.17032494605655257]]) + assert_array_almost_equal(actual, desired, decimal=15) + + random.seed(self.seed) + actual = random.randn() + assert_array_almost_equal(actual, desired[0, 0], decimal=15) + + def test_randint(self): + random.seed(self.seed) + actual = random.randint(-99, 99, size=(3, 2)) + desired = np.array([[31, 3], + [-52, 41], + [-48, -66]]) + assert_array_equal(actual, desired) + + def test_random_integers(self): + random.seed(self.seed) + with suppress_warnings() as sup: + w = sup.record(DeprecationWarning) + actual = random.random_integers(-99, 99, size=(3, 2)) + assert_(len(w) == 1) + desired = np.array([[31, 3], + [-52, 41], + [-48, -66]]) + assert_array_equal(actual, desired) + + random.seed(self.seed) + with suppress_warnings() as sup: + w = sup.record(DeprecationWarning) + actual = random.random_integers(198, size=(3, 2)) + assert_(len(w) == 1) + assert_array_equal(actual, desired + 100) + + def test_tomaxint(self): + random.seed(self.seed) + rs = random.RandomState(self.seed) + actual = rs.tomaxint(size=(3, 2)) + if np.iinfo(int).max == 2147483647: + desired = np.array([[1328851649, 731237375], + [1270502067, 320041495], + [1908433478, 499156889]], dtype=np.int64) + else: + desired = np.array([[5707374374421908479, 5456764827585442327], + [8196659375100692377, 8224063923314595285], + [4220315081820346526, 7177518203184491332]], + dtype=np.int64) + + assert_equal(actual, desired) + + rs.seed(self.seed) + actual = rs.tomaxint() + assert_equal(actual, desired[0, 0]) + + def test_random_integers_max_int(self): + # Tests whether random_integers can generate the + # maximum allowed Python int that can be converted + # into a C long. Previous implementations of this + # method have thrown an OverflowError when attempting + # to generate this integer. + with suppress_warnings() as sup: + w = sup.record(DeprecationWarning) + actual = random.random_integers(np.iinfo('l').max, + np.iinfo('l').max) + assert_(len(w) == 1) + + desired = np.iinfo('l').max + assert_equal(actual, desired) + with suppress_warnings() as sup: + w = sup.record(DeprecationWarning) + typer = np.dtype('l').type + actual = random.random_integers(typer(np.iinfo('l').max), + typer(np.iinfo('l').max)) + assert_(len(w) == 1) + assert_equal(actual, desired) + + def test_random_integers_deprecated(self): + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + + # DeprecationWarning raised with high == None + assert_raises(DeprecationWarning, + random.random_integers, + np.iinfo('l').max) + + # DeprecationWarning raised with high != None + assert_raises(DeprecationWarning, + random.random_integers, + np.iinfo('l').max, np.iinfo('l').max) + + def test_random_sample(self): + random.seed(self.seed) + actual = random.random_sample((3, 2)) + desired = np.array([[0.61879477158567997, 0.59162362775974664], + [0.88868358904449662, 0.89165480011560816], + [0.4575674820298663, 0.7781880808593471]]) + assert_array_almost_equal(actual, desired, decimal=15) + + random.seed(self.seed) + actual = random.random_sample() + assert_array_almost_equal(actual, desired[0, 0], decimal=15) + + def test_choice_uniform_replace(self): + random.seed(self.seed) + actual = random.choice(4, 4) + desired = np.array([2, 3, 2, 3]) + assert_array_equal(actual, desired) + + def test_choice_nonuniform_replace(self): + random.seed(self.seed) + actual = random.choice(4, 4, p=[0.4, 0.4, 0.1, 0.1]) + desired = np.array([1, 1, 2, 2]) + assert_array_equal(actual, desired) + + def test_choice_uniform_noreplace(self): + random.seed(self.seed) + actual = random.choice(4, 3, replace=False) + desired = np.array([0, 1, 3]) + assert_array_equal(actual, desired) + + def test_choice_nonuniform_noreplace(self): + random.seed(self.seed) + actual = random.choice(4, 3, replace=False, p=[0.1, 0.3, 0.5, 0.1]) + desired = np.array([2, 3, 1]) + assert_array_equal(actual, desired) + + def test_choice_noninteger(self): + random.seed(self.seed) + actual = random.choice(['a', 'b', 'c', 'd'], 4) + desired = np.array(['c', 'd', 'c', 'd']) + assert_array_equal(actual, desired) + + def test_choice_exceptions(self): + sample = random.choice + assert_raises(ValueError, sample, -1, 3) + assert_raises(ValueError, sample, 3., 3) + assert_raises(ValueError, sample, [[1, 2], [3, 4]], 3) + assert_raises(ValueError, sample, [], 3) + assert_raises(ValueError, sample, [1, 2, 3, 4], 3, + p=[[0.25, 0.25], [0.25, 0.25]]) + assert_raises(ValueError, sample, [1, 2], 3, p=[0.4, 0.4, 0.2]) + assert_raises(ValueError, sample, [1, 2], 3, p=[1.1, -0.1]) + assert_raises(ValueError, sample, [1, 2], 3, p=[0.4, 0.4]) + assert_raises(ValueError, sample, [1, 2, 3], 4, replace=False) + # gh-13087 + assert_raises(ValueError, sample, [1, 2, 3], -2, replace=False) + assert_raises(ValueError, sample, [1, 2, 3], (-1,), replace=False) + assert_raises(ValueError, sample, [1, 2, 3], (-1, 1), replace=False) + assert_raises(ValueError, sample, [1, 2, 3], 2, + replace=False, p=[1, 0, 0]) + + def test_choice_return_shape(self): + p = [0.1, 0.9] + # Check scalar + assert_(np.isscalar(random.choice(2, replace=True))) + assert_(np.isscalar(random.choice(2, replace=False))) + assert_(np.isscalar(random.choice(2, replace=True, p=p))) + assert_(np.isscalar(random.choice(2, replace=False, p=p))) + assert_(np.isscalar(random.choice([1, 2], replace=True))) + assert_(random.choice([None], replace=True) is None) + a = np.array([1, 2]) + arr = np.empty(1, dtype=object) + arr[0] = a + assert_(random.choice(arr, replace=True) is a) + + # Check 0-d array + s = tuple() + assert_(not np.isscalar(random.choice(2, s, replace=True))) + assert_(not np.isscalar(random.choice(2, s, replace=False))) + assert_(not np.isscalar(random.choice(2, s, replace=True, p=p))) + assert_(not np.isscalar(random.choice(2, s, replace=False, p=p))) + assert_(not np.isscalar(random.choice([1, 2], s, replace=True))) + assert_(random.choice([None], s, replace=True).ndim == 0) + a = np.array([1, 2]) + arr = np.empty(1, dtype=object) + arr[0] = a + assert_(random.choice(arr, s, replace=True).item() is a) + + # Check multi dimensional array + s = (2, 3) + p = [0.1, 0.1, 0.1, 0.1, 0.4, 0.2] + assert_equal(random.choice(6, s, replace=True).shape, s) + assert_equal(random.choice(6, s, replace=False).shape, s) + assert_equal(random.choice(6, s, replace=True, p=p).shape, s) + assert_equal(random.choice(6, s, replace=False, p=p).shape, s) + assert_equal(random.choice(np.arange(6), s, replace=True).shape, s) + + # Check zero-size + assert_equal(random.randint(0, 0, size=(3, 0, 4)).shape, (3, 0, 4)) + assert_equal(random.randint(0, -10, size=0).shape, (0,)) + assert_equal(random.randint(10, 10, size=0).shape, (0,)) + assert_equal(random.choice(0, size=0).shape, (0,)) + assert_equal(random.choice([], size=(0,)).shape, (0,)) + assert_equal(random.choice(['a', 'b'], size=(3, 0, 4)).shape, + (3, 0, 4)) + assert_raises(ValueError, random.choice, [], 10) + + def test_choice_nan_probabilities(self): + a = np.array([42, 1, 2]) + p = [None, None, None] + assert_raises(ValueError, random.choice, a, p=p) + + def test_choice_p_non_contiguous(self): + p = np.ones(10) / 5 + p[1::2] = 3.0 + random.seed(self.seed) + non_contig = random.choice(5, 3, p=p[::2]) + random.seed(self.seed) + contig = random.choice(5, 3, p=np.ascontiguousarray(p[::2])) + assert_array_equal(non_contig, contig) + + def test_bytes(self): + random.seed(self.seed) + actual = random.bytes(10) + desired = b'\x82Ui\x9e\xff\x97+Wf\xa5' + assert_equal(actual, desired) + + def test_shuffle(self): + # Test lists, arrays (of various dtypes), and multidimensional versions + # of both, c-contiguous or not: + for conv in [lambda x: np.array([]), + lambda x: x, + lambda x: np.asarray(x).astype(np.int8), + lambda x: np.asarray(x).astype(np.float32), + lambda x: np.asarray(x).astype(np.complex64), + lambda x: np.asarray(x).astype(object), + lambda x: [(i, i) for i in x], + lambda x: np.asarray([[i, i] for i in x]), + lambda x: np.vstack([x, x]).T, + # gh-11442 + lambda x: (np.asarray([(i, i) for i in x], + [("a", int), ("b", int)]) + .view(np.recarray)), + # gh-4270 + lambda x: np.asarray([(i, i) for i in x], + [("a", object, (1,)), + ("b", np.int32, (1,))])]: + random.seed(self.seed) + alist = conv([1, 2, 3, 4, 5, 6, 7, 8, 9, 0]) + random.shuffle(alist) + actual = alist + desired = conv([0, 1, 9, 6, 2, 4, 5, 8, 7, 3]) + assert_array_equal(actual, desired) + + def test_shuffle_masked(self): + # gh-3263 + a = np.ma.masked_values(np.reshape(range(20), (5, 4)) % 3 - 1, -1) + b = np.ma.masked_values(np.arange(20) % 3 - 1, -1) + a_orig = a.copy() + b_orig = b.copy() + for i in range(50): + random.shuffle(a) + assert_equal( + sorted(a.data[~a.mask]), sorted(a_orig.data[~a_orig.mask])) + random.shuffle(b) + assert_equal( + sorted(b.data[~b.mask]), sorted(b_orig.data[~b_orig.mask])) + + def test_shuffle_invalid_objects(self): + x = np.array(3) + assert_raises(TypeError, random.shuffle, x) + + def test_permutation(self): + random.seed(self.seed) + alist = [1, 2, 3, 4, 5, 6, 7, 8, 9, 0] + actual = random.permutation(alist) + desired = [0, 1, 9, 6, 2, 4, 5, 8, 7, 3] + assert_array_equal(actual, desired) + + random.seed(self.seed) + arr_2d = np.atleast_2d([1, 2, 3, 4, 5, 6, 7, 8, 9, 0]).T + actual = random.permutation(arr_2d) + assert_array_equal(actual, np.atleast_2d(desired).T) + + random.seed(self.seed) + bad_x_str = "abcd" + assert_raises(IndexError, random.permutation, bad_x_str) + + random.seed(self.seed) + bad_x_float = 1.2 + assert_raises(IndexError, random.permutation, bad_x_float) + + integer_val = 10 + desired = [9, 0, 8, 5, 1, 3, 4, 7, 6, 2] + + random.seed(self.seed) + actual = random.permutation(integer_val) + assert_array_equal(actual, desired) + + def test_beta(self): + random.seed(self.seed) + actual = random.beta(.1, .9, size=(3, 2)) + desired = np.array( + [[1.45341850513746058e-02, 5.31297615662868145e-04], + [1.85366619058432324e-06, 4.19214516800110563e-03], + [1.58405155108498093e-04, 1.26252891949397652e-04]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_binomial(self): + random.seed(self.seed) + actual = random.binomial(100.123, .456, size=(3, 2)) + desired = np.array([[37, 43], + [42, 48], + [46, 45]]) + assert_array_equal(actual, desired) + + random.seed(self.seed) + actual = random.binomial(100.123, .456) + desired = 37 + assert_array_equal(actual, desired) + + def test_chisquare(self): + random.seed(self.seed) + actual = random.chisquare(50, size=(3, 2)) + desired = np.array([[63.87858175501090585, 68.68407748911370447], + [65.77116116901505904, 47.09686762438974483], + [72.3828403199695174, 74.18408615260374006]]) + assert_array_almost_equal(actual, desired, decimal=13) + + def test_dirichlet(self): + random.seed(self.seed) + alpha = np.array([51.72840233779265162, 39.74494232180943953]) + actual = random.dirichlet(alpha, size=(3, 2)) + desired = np.array([[[0.54539444573611562, 0.45460555426388438], + [0.62345816822039413, 0.37654183177960598]], + [[0.55206000085785778, 0.44793999914214233], + [0.58964023305154301, 0.41035976694845688]], + [[0.59266909280647828, 0.40733090719352177], + [0.56974431743975207, 0.43025568256024799]]]) + assert_array_almost_equal(actual, desired, decimal=15) + bad_alpha = np.array([5.4e-01, -1.0e-16]) + assert_raises(ValueError, random.dirichlet, bad_alpha) + + random.seed(self.seed) + alpha = np.array([51.72840233779265162, 39.74494232180943953]) + actual = random.dirichlet(alpha) + assert_array_almost_equal(actual, desired[0, 0], decimal=15) + + def test_dirichlet_size(self): + # gh-3173 + p = np.array([51.72840233779265162, 39.74494232180943953]) + assert_equal(random.dirichlet(p, np.uint32(1)).shape, (1, 2)) + assert_equal(random.dirichlet(p, np.uint32(1)).shape, (1, 2)) + assert_equal(random.dirichlet(p, np.uint32(1)).shape, (1, 2)) + assert_equal(random.dirichlet(p, [2, 2]).shape, (2, 2, 2)) + assert_equal(random.dirichlet(p, (2, 2)).shape, (2, 2, 2)) + assert_equal(random.dirichlet(p, np.array((2, 2))).shape, (2, 2, 2)) + + assert_raises(TypeError, random.dirichlet, p, float(1)) + + def test_dirichlet_bad_alpha(self): + # gh-2089 + alpha = np.array([5.4e-01, -1.0e-16]) + assert_raises(ValueError, random.dirichlet, alpha) + + def test_dirichlet_alpha_non_contiguous(self): + a = np.array([51.72840233779265162, -1.0, 39.74494232180943953]) + alpha = a[::2] + random.seed(self.seed) + non_contig = random.dirichlet(alpha, size=(3, 2)) + random.seed(self.seed) + contig = random.dirichlet(np.ascontiguousarray(alpha), + size=(3, 2)) + assert_array_almost_equal(non_contig, contig) + + def test_exponential(self): + random.seed(self.seed) + actual = random.exponential(1.1234, size=(3, 2)) + desired = np.array([[1.08342649775011624, 1.00607889924557314], + [2.46628830085216721, 2.49668106809923884], + [0.68717433461363442, 1.69175666993575979]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_exponential_0(self): + assert_equal(random.exponential(scale=0), 0) + assert_raises(ValueError, random.exponential, scale=-0.) + + def test_f(self): + random.seed(self.seed) + actual = random.f(12, 77, size=(3, 2)) + desired = np.array([[1.21975394418575878, 1.75135759791559775], + [1.44803115017146489, 1.22108959480396262], + [1.02176975757740629, 1.34431827623300415]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_gamma(self): + random.seed(self.seed) + actual = random.gamma(5, 3, size=(3, 2)) + desired = np.array([[24.60509188649287182, 28.54993563207210627], + [26.13476110204064184, 12.56988482927716078], + [31.71863275789960568, 33.30143302795922011]]) + assert_array_almost_equal(actual, desired, decimal=14) + + def test_gamma_0(self): + assert_equal(random.gamma(shape=0, scale=0), 0) + assert_raises(ValueError, random.gamma, shape=-0., scale=-0.) + + def test_geometric(self): + random.seed(self.seed) + actual = random.geometric(.123456789, size=(3, 2)) + desired = np.array([[8, 7], + [17, 17], + [5, 12]]) + assert_array_equal(actual, desired) + + def test_geometric_exceptions(self): + assert_raises(ValueError, random.geometric, 1.1) + assert_raises(ValueError, random.geometric, [1.1] * 10) + assert_raises(ValueError, random.geometric, -0.1) + assert_raises(ValueError, random.geometric, [-0.1] * 10) + with suppress_warnings() as sup: + sup.record(RuntimeWarning) + assert_raises(ValueError, random.geometric, np.nan) + assert_raises(ValueError, random.geometric, [np.nan] * 10) + + def test_gumbel(self): + random.seed(self.seed) + actual = random.gumbel(loc=.123456789, scale=2.0, size=(3, 2)) + desired = np.array([[0.19591898743416816, 0.34405539668096674], + [-1.4492522252274278, -1.47374816298446865], + [1.10651090478803416, -0.69535848626236174]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_gumbel_0(self): + assert_equal(random.gumbel(scale=0), 0) + assert_raises(ValueError, random.gumbel, scale=-0.) + + def test_hypergeometric(self): + random.seed(self.seed) + actual = random.hypergeometric(10.1, 5.5, 14, size=(3, 2)) + desired = np.array([[10, 10], + [10, 10], + [9, 9]]) + assert_array_equal(actual, desired) + + # Test nbad = 0 + actual = random.hypergeometric(5, 0, 3, size=4) + desired = np.array([3, 3, 3, 3]) + assert_array_equal(actual, desired) + + actual = random.hypergeometric(15, 0, 12, size=4) + desired = np.array([12, 12, 12, 12]) + assert_array_equal(actual, desired) + + # Test ngood = 0 + actual = random.hypergeometric(0, 5, 3, size=4) + desired = np.array([0, 0, 0, 0]) + assert_array_equal(actual, desired) + + actual = random.hypergeometric(0, 15, 12, size=4) + desired = np.array([0, 0, 0, 0]) + assert_array_equal(actual, desired) + + def test_laplace(self): + random.seed(self.seed) + actual = random.laplace(loc=.123456789, scale=2.0, size=(3, 2)) + desired = np.array([[0.66599721112760157, 0.52829452552221945], + [3.12791959514407125, 3.18202813572992005], + [-0.05391065675859356, 1.74901336242837324]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_laplace_0(self): + assert_equal(random.laplace(scale=0), 0) + assert_raises(ValueError, random.laplace, scale=-0.) + + def test_logistic(self): + random.seed(self.seed) + actual = random.logistic(loc=.123456789, scale=2.0, size=(3, 2)) + desired = np.array([[1.09232835305011444, 0.8648196662399954], + [4.27818590694950185, 4.33897006346929714], + [-0.21682183359214885, 2.63373365386060332]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_lognormal(self): + random.seed(self.seed) + actual = random.lognormal(mean=.123456789, sigma=2.0, size=(3, 2)) + desired = np.array([[16.50698631688883822, 36.54846706092654784], + [22.67886599981281748, 0.71617561058995771], + [65.72798501792723869, 86.84341601437161273]]) + assert_array_almost_equal(actual, desired, decimal=13) + + def test_lognormal_0(self): + assert_equal(random.lognormal(sigma=0), 1) + assert_raises(ValueError, random.lognormal, sigma=-0.) + + def test_logseries(self): + random.seed(self.seed) + actual = random.logseries(p=.923456789, size=(3, 2)) + desired = np.array([[2, 2], + [6, 17], + [3, 6]]) + assert_array_equal(actual, desired) + + def test_logseries_zero(self): + assert random.logseries(0) == 1 + + @pytest.mark.parametrize("value", [np.nextafter(0., -1), 1., np.nan, 5.]) + def test_logseries_exceptions(self, value): + with np.errstate(invalid="ignore"): + with pytest.raises(ValueError): + random.logseries(value) + with pytest.raises(ValueError): + # contiguous path: + random.logseries(np.array([value] * 10)) + with pytest.raises(ValueError): + # non-contiguous path: + random.logseries(np.array([value] * 10)[::2]) + + def test_multinomial(self): + random.seed(self.seed) + actual = random.multinomial(20, [1 / 6.] * 6, size=(3, 2)) + desired = np.array([[[4, 3, 5, 4, 2, 2], + [5, 2, 8, 2, 2, 1]], + [[3, 4, 3, 6, 0, 4], + [2, 1, 4, 3, 6, 4]], + [[4, 4, 2, 5, 2, 3], + [4, 3, 4, 2, 3, 4]]]) + assert_array_equal(actual, desired) + + def test_multivariate_normal(self): + random.seed(self.seed) + mean = (.123456789, 10) + cov = [[1, 0], [0, 1]] + size = (3, 2) + actual = random.multivariate_normal(mean, cov, size) + desired = np.array([[[1.463620246718631, 11.73759122771936], + [1.622445133300628, 9.771356667546383]], + [[2.154490787682787, 12.170324946056553], + [1.719909438201865, 9.230548443648306]], + [[0.689515026297799, 9.880729819607714], + [-0.023054015651998, 9.201096623542879]]]) + + assert_array_almost_equal(actual, desired, decimal=15) + + # Check for default size, was raising deprecation warning + actual = random.multivariate_normal(mean, cov) + desired = np.array([0.895289569463708, 9.17180864067987]) + assert_array_almost_equal(actual, desired, decimal=15) + + # Check that non positive-semidefinite covariance warns with + # RuntimeWarning + mean = [0, 0] + cov = [[1, 2], [2, 1]] + assert_warns(RuntimeWarning, random.multivariate_normal, mean, cov) + + # and that it doesn't warn with RuntimeWarning check_valid='ignore' + assert_no_warnings(random.multivariate_normal, mean, cov, + check_valid='ignore') + + # and that it raises with RuntimeWarning check_valid='raises' + assert_raises(ValueError, random.multivariate_normal, mean, cov, + check_valid='raise') + + cov = np.array([[1, 0.1], [0.1, 1]], dtype=np.float32) + with suppress_warnings() as sup: + random.multivariate_normal(mean, cov) + w = sup.record(RuntimeWarning) + assert len(w) == 0 + + mu = np.zeros(2) + cov = np.eye(2) + assert_raises(ValueError, random.multivariate_normal, mean, cov, + check_valid='other') + assert_raises(ValueError, random.multivariate_normal, + np.zeros((2, 1, 1)), cov) + assert_raises(ValueError, random.multivariate_normal, + mu, np.empty((3, 2))) + assert_raises(ValueError, random.multivariate_normal, + mu, np.eye(3)) + + def test_negative_binomial(self): + random.seed(self.seed) + actual = random.negative_binomial(n=100, p=.12345, size=(3, 2)) + desired = np.array([[848, 841], + [892, 611], + [779, 647]]) + assert_array_equal(actual, desired) + + def test_negative_binomial_exceptions(self): + with suppress_warnings() as sup: + sup.record(RuntimeWarning) + assert_raises(ValueError, random.negative_binomial, 100, np.nan) + assert_raises(ValueError, random.negative_binomial, 100, + [np.nan] * 10) + + def test_noncentral_chisquare(self): + random.seed(self.seed) + actual = random.noncentral_chisquare(df=5, nonc=5, size=(3, 2)) + desired = np.array([[23.91905354498517511, 13.35324692733826346], + [31.22452661329736401, 16.60047399466177254], + [5.03461598262724586, 17.94973089023519464]]) + assert_array_almost_equal(actual, desired, decimal=14) + + actual = random.noncentral_chisquare(df=.5, nonc=.2, size=(3, 2)) + desired = np.array([[1.47145377828516666, 0.15052899268012659], + [0.00943803056963588, 1.02647251615666169], + [0.332334982684171, 0.15451287602753125]]) + assert_array_almost_equal(actual, desired, decimal=14) + + random.seed(self.seed) + actual = random.noncentral_chisquare(df=5, nonc=0, size=(3, 2)) + desired = np.array([[9.597154162763948, 11.725484450296079], + [10.413711048138335, 3.694475922923986], + [13.484222138963087, 14.377255424602957]]) + assert_array_almost_equal(actual, desired, decimal=14) + + def test_noncentral_f(self): + random.seed(self.seed) + actual = random.noncentral_f(dfnum=5, dfden=2, nonc=1, + size=(3, 2)) + desired = np.array([[1.40598099674926669, 0.34207973179285761], + [3.57715069265772545, 7.92632662577829805], + [0.43741599463544162, 1.1774208752428319]]) + assert_array_almost_equal(actual, desired, decimal=14) + + def test_noncentral_f_nan(self): + random.seed(self.seed) + actual = random.noncentral_f(dfnum=5, dfden=2, nonc=np.nan) + assert np.isnan(actual) + + def test_normal(self): + random.seed(self.seed) + actual = random.normal(loc=.123456789, scale=2.0, size=(3, 2)) + desired = np.array([[2.80378370443726244, 3.59863924443872163], + [3.121433477601256, -0.33382987590723379], + [4.18552478636557357, 4.46410668111310471]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_normal_0(self): + assert_equal(random.normal(scale=0), 0) + assert_raises(ValueError, random.normal, scale=-0.) + + def test_pareto(self): + random.seed(self.seed) + actual = random.pareto(a=.123456789, size=(3, 2)) + desired = np.array( + [[2.46852460439034849e+03, 1.41286880810518346e+03], + [5.28287797029485181e+07, 6.57720981047328785e+07], + [1.40840323350391515e+02, 1.98390255135251704e+05]]) + # For some reason on 32-bit x86 Ubuntu 12.10 the [1, 0] entry in this + # matrix differs by 24 nulps. Discussion: + # https://mail.python.org/pipermail/numpy-discussion/2012-September/063801.html + # Consensus is that this is probably some gcc quirk that affects + # rounding but not in any important way, so we just use a looser + # tolerance on this test: + np.testing.assert_array_almost_equal_nulp(actual, desired, nulp=30) + + def test_poisson(self): + random.seed(self.seed) + actual = random.poisson(lam=.123456789, size=(3, 2)) + desired = np.array([[0, 0], + [1, 0], + [0, 0]]) + assert_array_equal(actual, desired) + + def test_poisson_exceptions(self): + lambig = np.iinfo('l').max + lamneg = -1 + assert_raises(ValueError, random.poisson, lamneg) + assert_raises(ValueError, random.poisson, [lamneg] * 10) + assert_raises(ValueError, random.poisson, lambig) + assert_raises(ValueError, random.poisson, [lambig] * 10) + with suppress_warnings() as sup: + sup.record(RuntimeWarning) + assert_raises(ValueError, random.poisson, np.nan) + assert_raises(ValueError, random.poisson, [np.nan] * 10) + + def test_power(self): + random.seed(self.seed) + actual = random.power(a=.123456789, size=(3, 2)) + desired = np.array([[0.02048932883240791, 0.01424192241128213], + [0.38446073748535298, 0.39499689943484395], + [0.00177699707563439, 0.13115505880863756]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_rayleigh(self): + random.seed(self.seed) + actual = random.rayleigh(scale=10, size=(3, 2)) + desired = np.array([[13.8882496494248393, 13.383318339044731], + [20.95413364294492098, 21.08285015800712614], + [11.06066537006854311, 17.35468505778271009]]) + assert_array_almost_equal(actual, desired, decimal=14) + + def test_rayleigh_0(self): + assert_equal(random.rayleigh(scale=0), 0) + assert_raises(ValueError, random.rayleigh, scale=-0.) + + def test_standard_cauchy(self): + random.seed(self.seed) + actual = random.standard_cauchy(size=(3, 2)) + desired = np.array([[0.77127660196445336, -6.55601161955910605], + [0.93582023391158309, -2.07479293013759447], + [-4.74601644297011926, 0.18338989290760804]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_standard_exponential(self): + random.seed(self.seed) + actual = random.standard_exponential(size=(3, 2)) + desired = np.array([[0.96441739162374596, 0.89556604882105506], + [2.1953785836319808, 2.22243285392490542], + [0.6116915921431676, 1.50592546727413201]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_standard_gamma(self): + random.seed(self.seed) + actual = random.standard_gamma(shape=3, size=(3, 2)) + desired = np.array([[5.50841531318455058, 6.62953470301903103], + [5.93988484943779227, 2.31044849402133989], + [7.54838614231317084, 8.012756093271868]]) + assert_array_almost_equal(actual, desired, decimal=14) + + def test_standard_gamma_0(self): + assert_equal(random.standard_gamma(shape=0), 0) + assert_raises(ValueError, random.standard_gamma, shape=-0.) + + def test_standard_normal(self): + random.seed(self.seed) + actual = random.standard_normal(size=(3, 2)) + desired = np.array([[1.34016345771863121, 1.73759122771936081], + [1.498988344300628, -0.2286433324536169], + [2.031033998682787, 2.17032494605655257]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_randn_singleton(self): + random.seed(self.seed) + actual = random.randn() + desired = np.array(1.34016345771863121) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_standard_t(self): + random.seed(self.seed) + actual = random.standard_t(df=10, size=(3, 2)) + desired = np.array([[0.97140611862659965, -0.08830486548450577], + [1.36311143689505321, -0.55317463909867071], + [-0.18473749069684214, 0.61181537341755321]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_triangular(self): + random.seed(self.seed) + actual = random.triangular(left=5.12, mode=10.23, right=20.34, + size=(3, 2)) + desired = np.array([[12.68117178949215784, 12.4129206149193152], + [16.20131377335158263, 16.25692138747600524], + [11.20400690911820263, 14.4978144835829923]]) + assert_array_almost_equal(actual, desired, decimal=14) + + def test_uniform(self): + random.seed(self.seed) + actual = random.uniform(low=1.23, high=10.54, size=(3, 2)) + desired = np.array([[6.99097932346268003, 6.73801597444323974], + [9.50364421400426274, 9.53130618907631089], + [5.48995325769805476, 8.47493103280052118]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_uniform_range_bounds(self): + fmin = np.finfo('float').min + fmax = np.finfo('float').max + + func = random.uniform + assert_raises(OverflowError, func, -np.inf, 0) + assert_raises(OverflowError, func, 0, np.inf) + assert_raises(OverflowError, func, fmin, fmax) + assert_raises(OverflowError, func, [-np.inf], [0]) + assert_raises(OverflowError, func, [0], [np.inf]) + + # (fmax / 1e17) - fmin is within range, so this should not throw + # account for i386 extended precision DBL_MAX / 1e17 + DBL_MAX > + # DBL_MAX by increasing fmin a bit + random.uniform(low=np.nextafter(fmin, 1), high=fmax / 1e17) + + def test_scalar_exception_propagation(self): + # Tests that exceptions are correctly propagated in distributions + # when called with objects that throw exceptions when converted to + # scalars. + # + # Regression test for gh: 8865 + + class ThrowingFloat(np.ndarray): + def __float__(self): + raise TypeError + + throwing_float = np.array(1.0).view(ThrowingFloat) + assert_raises(TypeError, random.uniform, throwing_float, + throwing_float) + + class ThrowingInteger(np.ndarray): + def __int__(self): + raise TypeError + + throwing_int = np.array(1).view(ThrowingInteger) + assert_raises(TypeError, random.hypergeometric, throwing_int, 1, 1) + + def test_vonmises(self): + random.seed(self.seed) + actual = random.vonmises(mu=1.23, kappa=1.54, size=(3, 2)) + desired = np.array([[2.28567572673902042, 2.89163838442285037], + [0.38198375564286025, 2.57638023113890746], + [1.19153771588353052, 1.83509849681825354]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_vonmises_small(self): + # check infinite loop, gh-4720 + random.seed(self.seed) + r = random.vonmises(mu=0., kappa=1.1e-8, size=10**6) + assert_(np.isfinite(r).all()) + + def test_vonmises_large(self): + # guard against changes in RandomState when Generator is fixed + random.seed(self.seed) + actual = random.vonmises(mu=0., kappa=1e7, size=3) + desired = np.array([4.634253748521111e-04, + 3.558873596114509e-04, + -2.337119622577433e-04]) + assert_array_almost_equal(actual, desired, decimal=8) + + def test_vonmises_nan(self): + random.seed(self.seed) + r = random.vonmises(mu=0., kappa=np.nan) + assert_(np.isnan(r)) + + def test_wald(self): + random.seed(self.seed) + actual = random.wald(mean=1.23, scale=1.54, size=(3, 2)) + desired = np.array([[3.82935265715889983, 5.13125249184285526], + [0.35045403618358717, 1.50832396872003538], + [0.24124319895843183, 0.22031101461955038]]) + assert_array_almost_equal(actual, desired, decimal=14) + + def test_weibull(self): + random.seed(self.seed) + actual = random.weibull(a=1.23, size=(3, 2)) + desired = np.array([[0.97097342648766727, 0.91422896443565516], + [1.89517770034962929, 1.91414357960479564], + [0.67057783752390987, 1.39494046635066793]]) + assert_array_almost_equal(actual, desired, decimal=15) + + def test_weibull_0(self): + random.seed(self.seed) + assert_equal(random.weibull(a=0, size=12), np.zeros(12)) + assert_raises(ValueError, random.weibull, a=-0.) + + def test_zipf(self): + random.seed(self.seed) + actual = random.zipf(a=1.23, size=(3, 2)) + desired = np.array([[66, 29], + [1, 1], + [3, 13]]) + assert_array_equal(actual, desired) + + +class TestBroadcast: + # tests that functions that broadcast behave + # correctly when presented with non-scalar arguments + def setup_method(self): + self.seed = 123456789 + + def set_seed(self): + random.seed(self.seed) + + def test_uniform(self): + low = [0] + high = [1] + uniform = random.uniform + desired = np.array([0.53283302478975902, + 0.53413660089041659, + 0.50955303552646702]) + + self.set_seed() + actual = uniform(low * 3, high) + assert_array_almost_equal(actual, desired, decimal=14) + + self.set_seed() + actual = uniform(low, high * 3) + assert_array_almost_equal(actual, desired, decimal=14) + + def test_normal(self): + loc = [0] + scale = [1] + bad_scale = [-1] + normal = random.normal + desired = np.array([2.2129019979039612, + 2.1283977976520019, + 1.8417114045748335]) + + self.set_seed() + actual = normal(loc * 3, scale) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, normal, loc * 3, bad_scale) + + self.set_seed() + actual = normal(loc, scale * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, normal, loc, bad_scale * 3) + + def test_beta(self): + a = [1] + b = [2] + bad_a = [-1] + bad_b = [-2] + beta = random.beta + desired = np.array([0.19843558305989056, + 0.075230336409423643, + 0.24976865978980844]) + + self.set_seed() + actual = beta(a * 3, b) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, beta, bad_a * 3, b) + assert_raises(ValueError, beta, a * 3, bad_b) + + self.set_seed() + actual = beta(a, b * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, beta, bad_a, b * 3) + assert_raises(ValueError, beta, a, bad_b * 3) + + def test_exponential(self): + scale = [1] + bad_scale = [-1] + exponential = random.exponential + desired = np.array([0.76106853658845242, + 0.76386282278691653, + 0.71243813125891797]) + + self.set_seed() + actual = exponential(scale * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, exponential, bad_scale * 3) + + def test_standard_gamma(self): + shape = [1] + bad_shape = [-1] + std_gamma = random.standard_gamma + desired = np.array([0.76106853658845242, + 0.76386282278691653, + 0.71243813125891797]) + + self.set_seed() + actual = std_gamma(shape * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, std_gamma, bad_shape * 3) + + def test_gamma(self): + shape = [1] + scale = [2] + bad_shape = [-1] + bad_scale = [-2] + gamma = random.gamma + desired = np.array([1.5221370731769048, + 1.5277256455738331, + 1.4248762625178359]) + + self.set_seed() + actual = gamma(shape * 3, scale) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, gamma, bad_shape * 3, scale) + assert_raises(ValueError, gamma, shape * 3, bad_scale) + + self.set_seed() + actual = gamma(shape, scale * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, gamma, bad_shape, scale * 3) + assert_raises(ValueError, gamma, shape, bad_scale * 3) + + def test_f(self): + dfnum = [1] + dfden = [2] + bad_dfnum = [-1] + bad_dfden = [-2] + f = random.f + desired = np.array([0.80038951638264799, + 0.86768719635363512, + 2.7251095168386801]) + + self.set_seed() + actual = f(dfnum * 3, dfden) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, f, bad_dfnum * 3, dfden) + assert_raises(ValueError, f, dfnum * 3, bad_dfden) + + self.set_seed() + actual = f(dfnum, dfden * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, f, bad_dfnum, dfden * 3) + assert_raises(ValueError, f, dfnum, bad_dfden * 3) + + def test_noncentral_f(self): + dfnum = [2] + dfden = [3] + nonc = [4] + bad_dfnum = [0] + bad_dfden = [-1] + bad_nonc = [-2] + nonc_f = random.noncentral_f + desired = np.array([9.1393943263705211, + 13.025456344595602, + 8.8018098359100545]) + + self.set_seed() + actual = nonc_f(dfnum * 3, dfden, nonc) + assert_array_almost_equal(actual, desired, decimal=14) + assert np.all(np.isnan(nonc_f(dfnum, dfden, [np.nan] * 3))) + + assert_raises(ValueError, nonc_f, bad_dfnum * 3, dfden, nonc) + assert_raises(ValueError, nonc_f, dfnum * 3, bad_dfden, nonc) + assert_raises(ValueError, nonc_f, dfnum * 3, dfden, bad_nonc) + + self.set_seed() + actual = nonc_f(dfnum, dfden * 3, nonc) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, nonc_f, bad_dfnum, dfden * 3, nonc) + assert_raises(ValueError, nonc_f, dfnum, bad_dfden * 3, nonc) + assert_raises(ValueError, nonc_f, dfnum, dfden * 3, bad_nonc) + + self.set_seed() + actual = nonc_f(dfnum, dfden, nonc * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, nonc_f, bad_dfnum, dfden, nonc * 3) + assert_raises(ValueError, nonc_f, dfnum, bad_dfden, nonc * 3) + assert_raises(ValueError, nonc_f, dfnum, dfden, bad_nonc * 3) + + def test_noncentral_f_small_df(self): + self.set_seed() + desired = np.array([6.869638627492048, 0.785880199263955]) + actual = random.noncentral_f(0.9, 0.9, 2, size=2) + assert_array_almost_equal(actual, desired, decimal=14) + + def test_chisquare(self): + df = [1] + bad_df = [-1] + chisquare = random.chisquare + desired = np.array([0.57022801133088286, + 0.51947702108840776, + 0.1320969254923558]) + + self.set_seed() + actual = chisquare(df * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, chisquare, bad_df * 3) + + def test_noncentral_chisquare(self): + df = [1] + nonc = [2] + bad_df = [-1] + bad_nonc = [-2] + nonc_chi = random.noncentral_chisquare + desired = np.array([9.0015599467913763, + 4.5804135049718742, + 6.0872302432834564]) + + self.set_seed() + actual = nonc_chi(df * 3, nonc) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, nonc_chi, bad_df * 3, nonc) + assert_raises(ValueError, nonc_chi, df * 3, bad_nonc) + + self.set_seed() + actual = nonc_chi(df, nonc * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, nonc_chi, bad_df, nonc * 3) + assert_raises(ValueError, nonc_chi, df, bad_nonc * 3) + + def test_standard_t(self): + df = [1] + bad_df = [-1] + t = random.standard_t + desired = np.array([3.0702872575217643, + 5.8560725167361607, + 1.0274791436474273]) + + self.set_seed() + actual = t(df * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, t, bad_df * 3) + assert_raises(ValueError, random.standard_t, bad_df * 3) + + def test_vonmises(self): + mu = [2] + kappa = [1] + bad_kappa = [-1] + vonmises = random.vonmises + desired = np.array([2.9883443664201312, + -2.7064099483995943, + -1.8672476700665914]) + + self.set_seed() + actual = vonmises(mu * 3, kappa) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, vonmises, mu * 3, bad_kappa) + + self.set_seed() + actual = vonmises(mu, kappa * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, vonmises, mu, bad_kappa * 3) + + def test_pareto(self): + a = [1] + bad_a = [-1] + pareto = random.pareto + desired = np.array([1.1405622680198362, + 1.1465519762044529, + 1.0389564467453547]) + + self.set_seed() + actual = pareto(a * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, pareto, bad_a * 3) + assert_raises(ValueError, random.pareto, bad_a * 3) + + def test_weibull(self): + a = [1] + bad_a = [-1] + weibull = random.weibull + desired = np.array([0.76106853658845242, + 0.76386282278691653, + 0.71243813125891797]) + + self.set_seed() + actual = weibull(a * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, weibull, bad_a * 3) + assert_raises(ValueError, random.weibull, bad_a * 3) + + def test_power(self): + a = [1] + bad_a = [-1] + power = random.power + desired = np.array([0.53283302478975902, + 0.53413660089041659, + 0.50955303552646702]) + + self.set_seed() + actual = power(a * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, power, bad_a * 3) + assert_raises(ValueError, random.power, bad_a * 3) + + def test_laplace(self): + loc = [0] + scale = [1] + bad_scale = [-1] + laplace = random.laplace + desired = np.array([0.067921356028507157, + 0.070715642226971326, + 0.019290950698972624]) + + self.set_seed() + actual = laplace(loc * 3, scale) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, laplace, loc * 3, bad_scale) + + self.set_seed() + actual = laplace(loc, scale * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, laplace, loc, bad_scale * 3) + + def test_gumbel(self): + loc = [0] + scale = [1] + bad_scale = [-1] + gumbel = random.gumbel + desired = np.array([0.2730318639556768, + 0.26936705726291116, + 0.33906220393037939]) + + self.set_seed() + actual = gumbel(loc * 3, scale) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, gumbel, loc * 3, bad_scale) + + self.set_seed() + actual = gumbel(loc, scale * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, gumbel, loc, bad_scale * 3) + + def test_logistic(self): + loc = [0] + scale = [1] + bad_scale = [-1] + logistic = random.logistic + desired = np.array([0.13152135837586171, + 0.13675915696285773, + 0.038216792802833396]) + + self.set_seed() + actual = logistic(loc * 3, scale) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, logistic, loc * 3, bad_scale) + + self.set_seed() + actual = logistic(loc, scale * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, logistic, loc, bad_scale * 3) + assert_equal(random.logistic(1.0, 0.0), 1.0) + + def test_lognormal(self): + mean = [0] + sigma = [1] + bad_sigma = [-1] + lognormal = random.lognormal + desired = np.array([9.1422086044848427, + 8.4013952870126261, + 6.3073234116578671]) + + self.set_seed() + actual = lognormal(mean * 3, sigma) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, lognormal, mean * 3, bad_sigma) + assert_raises(ValueError, random.lognormal, mean * 3, bad_sigma) + + self.set_seed() + actual = lognormal(mean, sigma * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, lognormal, mean, bad_sigma * 3) + assert_raises(ValueError, random.lognormal, mean, bad_sigma * 3) + + def test_rayleigh(self): + scale = [1] + bad_scale = [-1] + rayleigh = random.rayleigh + desired = np.array([1.2337491937897689, + 1.2360119924878694, + 1.1936818095781789]) + + self.set_seed() + actual = rayleigh(scale * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, rayleigh, bad_scale * 3) + + def test_wald(self): + mean = [0.5] + scale = [1] + bad_mean = [0] + bad_scale = [-2] + wald = random.wald + desired = np.array([0.11873681120271318, + 0.12450084820795027, + 0.9096122728408238]) + + self.set_seed() + actual = wald(mean * 3, scale) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, wald, bad_mean * 3, scale) + assert_raises(ValueError, wald, mean * 3, bad_scale) + assert_raises(ValueError, random.wald, bad_mean * 3, scale) + assert_raises(ValueError, random.wald, mean * 3, bad_scale) + + self.set_seed() + actual = wald(mean, scale * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, wald, bad_mean, scale * 3) + assert_raises(ValueError, wald, mean, bad_scale * 3) + assert_raises(ValueError, wald, 0.0, 1) + assert_raises(ValueError, wald, 0.5, 0.0) + + def test_triangular(self): + left = [1] + right = [3] + mode = [2] + bad_left_one = [3] + bad_mode_one = [4] + bad_left_two, bad_mode_two = right * 2 + triangular = random.triangular + desired = np.array([2.03339048710429, + 2.0347400359389356, + 2.0095991069536208]) + + self.set_seed() + actual = triangular(left * 3, mode, right) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, triangular, bad_left_one * 3, mode, right) + assert_raises(ValueError, triangular, left * 3, bad_mode_one, right) + assert_raises(ValueError, triangular, bad_left_two * 3, bad_mode_two, + right) + + self.set_seed() + actual = triangular(left, mode * 3, right) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, triangular, bad_left_one, mode * 3, right) + assert_raises(ValueError, triangular, left, bad_mode_one * 3, right) + assert_raises(ValueError, triangular, bad_left_two, bad_mode_two * 3, + right) + + self.set_seed() + actual = triangular(left, mode, right * 3) + assert_array_almost_equal(actual, desired, decimal=14) + assert_raises(ValueError, triangular, bad_left_one, mode, right * 3) + assert_raises(ValueError, triangular, left, bad_mode_one, right * 3) + assert_raises(ValueError, triangular, bad_left_two, bad_mode_two, + right * 3) + + assert_raises(ValueError, triangular, 10., 0., 20.) + assert_raises(ValueError, triangular, 10., 25., 20.) + assert_raises(ValueError, triangular, 10., 10., 10.) + + def test_binomial(self): + n = [1] + p = [0.5] + bad_n = [-1] + bad_p_one = [-1] + bad_p_two = [1.5] + binom = random.binomial + desired = np.array([1, 1, 1]) + + self.set_seed() + actual = binom(n * 3, p) + assert_array_equal(actual, desired) + assert_raises(ValueError, binom, bad_n * 3, p) + assert_raises(ValueError, binom, n * 3, bad_p_one) + assert_raises(ValueError, binom, n * 3, bad_p_two) + + self.set_seed() + actual = binom(n, p * 3) + assert_array_equal(actual, desired) + assert_raises(ValueError, binom, bad_n, p * 3) + assert_raises(ValueError, binom, n, bad_p_one * 3) + assert_raises(ValueError, binom, n, bad_p_two * 3) + + def test_negative_binomial(self): + n = [1] + p = [0.5] + bad_n = [-1] + bad_p_one = [-1] + bad_p_two = [1.5] + neg_binom = random.negative_binomial + desired = np.array([1, 0, 1]) + + self.set_seed() + actual = neg_binom(n * 3, p) + assert_array_equal(actual, desired) + assert_raises(ValueError, neg_binom, bad_n * 3, p) + assert_raises(ValueError, neg_binom, n * 3, bad_p_one) + assert_raises(ValueError, neg_binom, n * 3, bad_p_two) + + self.set_seed() + actual = neg_binom(n, p * 3) + assert_array_equal(actual, desired) + assert_raises(ValueError, neg_binom, bad_n, p * 3) + assert_raises(ValueError, neg_binom, n, bad_p_one * 3) + assert_raises(ValueError, neg_binom, n, bad_p_two * 3) + + def test_poisson(self): + max_lam = random.RandomState()._poisson_lam_max + + lam = [1] + bad_lam_one = [-1] + bad_lam_two = [max_lam * 2] + poisson = random.poisson + desired = np.array([1, 1, 0]) + + self.set_seed() + actual = poisson(lam * 3) + assert_array_equal(actual, desired) + assert_raises(ValueError, poisson, bad_lam_one * 3) + assert_raises(ValueError, poisson, bad_lam_two * 3) + + def test_zipf(self): + a = [2] + bad_a = [0] + zipf = random.zipf + desired = np.array([2, 2, 1]) + + self.set_seed() + actual = zipf(a * 3) + assert_array_equal(actual, desired) + assert_raises(ValueError, zipf, bad_a * 3) + with np.errstate(invalid='ignore'): + assert_raises(ValueError, zipf, np.nan) + assert_raises(ValueError, zipf, [0, 0, np.nan]) + + def test_geometric(self): + p = [0.5] + bad_p_one = [-1] + bad_p_two = [1.5] + geom = random.geometric + desired = np.array([2, 2, 2]) + + self.set_seed() + actual = geom(p * 3) + assert_array_equal(actual, desired) + assert_raises(ValueError, geom, bad_p_one * 3) + assert_raises(ValueError, geom, bad_p_two * 3) + + def test_hypergeometric(self): + ngood = [1] + nbad = [2] + nsample = [2] + bad_ngood = [-1] + bad_nbad = [-2] + bad_nsample_one = [0] + bad_nsample_two = [4] + hypergeom = random.hypergeometric + desired = np.array([1, 1, 1]) + + self.set_seed() + actual = hypergeom(ngood * 3, nbad, nsample) + assert_array_equal(actual, desired) + assert_raises(ValueError, hypergeom, bad_ngood * 3, nbad, nsample) + assert_raises(ValueError, hypergeom, ngood * 3, bad_nbad, nsample) + assert_raises(ValueError, hypergeom, ngood * 3, nbad, bad_nsample_one) + assert_raises(ValueError, hypergeom, ngood * 3, nbad, bad_nsample_two) + + self.set_seed() + actual = hypergeom(ngood, nbad * 3, nsample) + assert_array_equal(actual, desired) + assert_raises(ValueError, hypergeom, bad_ngood, nbad * 3, nsample) + assert_raises(ValueError, hypergeom, ngood, bad_nbad * 3, nsample) + assert_raises(ValueError, hypergeom, ngood, nbad * 3, bad_nsample_one) + assert_raises(ValueError, hypergeom, ngood, nbad * 3, bad_nsample_two) + + self.set_seed() + actual = hypergeom(ngood, nbad, nsample * 3) + assert_array_equal(actual, desired) + assert_raises(ValueError, hypergeom, bad_ngood, nbad, nsample * 3) + assert_raises(ValueError, hypergeom, ngood, bad_nbad, nsample * 3) + assert_raises(ValueError, hypergeom, ngood, nbad, bad_nsample_one * 3) + assert_raises(ValueError, hypergeom, ngood, nbad, bad_nsample_two * 3) + + assert_raises(ValueError, hypergeom, -1, 10, 20) + assert_raises(ValueError, hypergeom, 10, -1, 20) + assert_raises(ValueError, hypergeom, 10, 10, 0) + assert_raises(ValueError, hypergeom, 10, 10, 25) + + def test_logseries(self): + p = [0.5] + bad_p_one = [2] + bad_p_two = [-1] + logseries = random.logseries + desired = np.array([1, 1, 1]) + + self.set_seed() + actual = logseries(p * 3) + assert_array_equal(actual, desired) + assert_raises(ValueError, logseries, bad_p_one * 3) + assert_raises(ValueError, logseries, bad_p_two * 3) + + +@pytest.mark.skipif(IS_WASM, reason="can't start thread") +class TestThread: + # make sure each state produces the same sequence even in threads + def setup_method(self): + self.seeds = range(4) + + def check_function(self, function, sz): + from threading import Thread + + out1 = np.empty((len(self.seeds),) + sz) + out2 = np.empty((len(self.seeds),) + sz) + + # threaded generation + t = [Thread(target=function, args=(random.RandomState(s), o)) + for s, o in zip(self.seeds, out1)] + [x.start() for x in t] + [x.join() for x in t] + + # the same serial + for s, o in zip(self.seeds, out2): + function(random.RandomState(s), o) + + # these platforms change x87 fpu precision mode in threads + if np.intp().dtype.itemsize == 4 and sys.platform == "win32": + assert_array_almost_equal(out1, out2) + else: + assert_array_equal(out1, out2) + + def test_normal(self): + def gen_random(state, out): + out[...] = state.normal(size=10000) + + self.check_function(gen_random, sz=(10000,)) + + def test_exp(self): + def gen_random(state, out): + out[...] = state.exponential(scale=np.ones((100, 1000))) + + self.check_function(gen_random, sz=(100, 1000)) + + def test_multinomial(self): + def gen_random(state, out): + out[...] = state.multinomial(10, [1 / 6.] * 6, size=10000) + + self.check_function(gen_random, sz=(10000, 6)) + + +# See Issue #4263 +class TestSingleEltArrayInput: + def setup_method(self): + self.argOne = np.array([2]) + self.argTwo = np.array([3]) + self.argThree = np.array([4]) + self.tgtShape = (1,) + + def test_one_arg_funcs(self): + funcs = (random.exponential, random.standard_gamma, + random.chisquare, random.standard_t, + random.pareto, random.weibull, + random.power, random.rayleigh, + random.poisson, random.zipf, + random.geometric, random.logseries) + + probfuncs = (random.geometric, random.logseries) + + for func in funcs: + if func in probfuncs: # p < 1.0 + out = func(np.array([0.5])) + + else: + out = func(self.argOne) + + assert_equal(out.shape, self.tgtShape) + + def test_two_arg_funcs(self): + funcs = (random.uniform, random.normal, + random.beta, random.gamma, + random.f, random.noncentral_chisquare, + random.vonmises, random.laplace, + random.gumbel, random.logistic, + random.lognormal, random.wald, + random.binomial, random.negative_binomial) + + probfuncs = (random.binomial, random.negative_binomial) + + for func in funcs: + if func in probfuncs: # p <= 1 + argTwo = np.array([0.5]) + + else: + argTwo = self.argTwo + + out = func(self.argOne, argTwo) + assert_equal(out.shape, self.tgtShape) + + out = func(self.argOne[0], argTwo) + assert_equal(out.shape, self.tgtShape) + + out = func(self.argOne, argTwo[0]) + assert_equal(out.shape, self.tgtShape) + + def test_three_arg_funcs(self): + funcs = [random.noncentral_f, random.triangular, + random.hypergeometric] + + for func in funcs: + out = func(self.argOne, self.argTwo, self.argThree) + assert_equal(out.shape, self.tgtShape) + + out = func(self.argOne[0], self.argTwo, self.argThree) + assert_equal(out.shape, self.tgtShape) + + out = func(self.argOne, self.argTwo[0], self.argThree) + assert_equal(out.shape, self.tgtShape) + + +# Ensure returned array dtype is correct for platform +def test_integer_dtype(int_func): + random.seed(123456789) + fname, args, sha256 = int_func + f = getattr(random, fname) + actual = f(*args, size=2) + assert_(actual.dtype == np.dtype('l')) + + +def test_integer_repeat(int_func): + random.seed(123456789) + fname, args, sha256 = int_func + f = getattr(random, fname) + val = f(*args, size=1000000) + if sys.byteorder != 'little': + val = val.byteswap() + res = hashlib.sha256(val.view(np.int8)).hexdigest() + assert_(res == sha256) + + +def test_broadcast_size_error(): + # GH-16833 + with pytest.raises(ValueError): + random.binomial(1, [0.3, 0.7], size=(2, 1)) + with pytest.raises(ValueError): + random.binomial([1, 2], 0.3, size=(2, 1)) + with pytest.raises(ValueError): + random.binomial([1, 2], [0.3, 0.7], size=(2, 1)) + + +def test_randomstate_ctor_old_style_pickle(): + rs = np.random.RandomState(MT19937(0)) + rs.standard_normal(1) + # Directly call reduce which is used in pickling + ctor, args, state_a = rs.__reduce__() + # Simulate unpickling an old pickle that only has the name + assert args[:1] == ("MT19937",) + b = ctor(*args[:1]) + b.set_state(state_a) + state_b = b.get_state(legacy=False) + + assert_equal(state_a['bit_generator'], state_b['bit_generator']) + assert_array_equal(state_a['state']['key'], state_b['state']['key']) + assert_array_equal(state_a['state']['pos'], state_b['state']['pos']) + assert_equal(state_a['has_gauss'], state_b['has_gauss']) + assert_equal(state_a['gauss'], state_b['gauss']) + + +def test_hot_swap(restore_singleton_bitgen): + # GH 21808 + def_bg = np.random.default_rng(0) + bg = def_bg.bit_generator + np.random.set_bit_generator(bg) + assert isinstance(np.random.mtrand._rand._bit_generator, type(bg)) + + second_bg = np.random.get_bit_generator() + assert bg is second_bg + + +def test_seed_alt_bit_gen(restore_singleton_bitgen): + # GH 21808 + bg = PCG64(0) + np.random.set_bit_generator(bg) + state = np.random.get_state(legacy=False) + np.random.seed(1) + new_state = np.random.get_state(legacy=False) + print(state) + print(new_state) + assert state["bit_generator"] == "PCG64" + assert state["state"]["state"] != new_state["state"]["state"] + assert state["state"]["inc"] != new_state["state"]["inc"] + + +def test_state_error_alt_bit_gen(restore_singleton_bitgen): + # GH 21808 + state = np.random.get_state() + bg = PCG64(0) + np.random.set_bit_generator(bg) + with pytest.raises(ValueError, match="state must be for a PCG64"): + np.random.set_state(state) + + +def test_swap_worked(restore_singleton_bitgen): + # GH 21808 + np.random.seed(98765) + vals = np.random.randint(0, 2 ** 30, 10) + bg = PCG64(0) + state = bg.state + np.random.set_bit_generator(bg) + state_direct = np.random.get_state(legacy=False) + for field in state: + assert state[field] == state_direct[field] + np.random.seed(98765) + pcg_vals = np.random.randint(0, 2 ** 30, 10) + assert not np.all(vals == pcg_vals) + new_state = bg.state + assert new_state["state"]["state"] != state["state"]["state"] + assert new_state["state"]["inc"] == new_state["state"]["inc"] + + +def test_swapped_singleton_against_direct(restore_singleton_bitgen): + np.random.set_bit_generator(PCG64(98765)) + singleton_vals = np.random.randint(0, 2 ** 30, 10) + rg = np.random.RandomState(PCG64(98765)) + non_singleton_vals = rg.randint(0, 2 ** 30, 10) + assert_equal(non_singleton_vals, singleton_vals) diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/test_randomstate_regression.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/test_randomstate_regression.py new file mode 100644 index 0000000000000000000000000000000000000000..7ad19ab5562b87305a0391c80c602259816a984e --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/test_randomstate_regression.py @@ -0,0 +1,216 @@ +import sys + +import pytest + +from numpy.testing import ( + assert_, assert_array_equal, assert_raises, + ) +import numpy as np + +from numpy import random + + +class TestRegression: + + def test_VonMises_range(self): + # Make sure generated random variables are in [-pi, pi]. + # Regression test for ticket #986. + for mu in np.linspace(-7., 7., 5): + r = random.vonmises(mu, 1, 50) + assert_(np.all(r > -np.pi) and np.all(r <= np.pi)) + + def test_hypergeometric_range(self): + # Test for ticket #921 + assert_(np.all(random.hypergeometric(3, 18, 11, size=10) < 4)) + assert_(np.all(random.hypergeometric(18, 3, 11, size=10) > 0)) + + # Test for ticket #5623 + args = [ + (2**20 - 2, 2**20 - 2, 2**20 - 2), # Check for 32-bit systems + ] + is_64bits = sys.maxsize > 2**32 + if is_64bits and sys.platform != 'win32': + # Check for 64-bit systems + args.append((2**40 - 2, 2**40 - 2, 2**40 - 2)) + for arg in args: + assert_(random.hypergeometric(*arg) > 0) + + def test_logseries_convergence(self): + # Test for ticket #923 + N = 1000 + random.seed(0) + rvsn = random.logseries(0.8, size=N) + # these two frequency counts should be close to theoretical + # numbers with this large sample + # theoretical large N result is 0.49706795 + freq = np.sum(rvsn == 1) / N + msg = f'Frequency was {freq:f}, should be > 0.45' + assert_(freq > 0.45, msg) + # theoretical large N result is 0.19882718 + freq = np.sum(rvsn == 2) / N + msg = f'Frequency was {freq:f}, should be < 0.23' + assert_(freq < 0.23, msg) + + def test_shuffle_mixed_dimension(self): + # Test for trac ticket #2074 + for t in [[1, 2, 3, None], + [(1, 1), (2, 2), (3, 3), None], + [1, (2, 2), (3, 3), None], + [(1, 1), 2, 3, None]]: + random.seed(12345) + shuffled = list(t) + random.shuffle(shuffled) + expected = np.array([t[0], t[3], t[1], t[2]], dtype=object) + assert_array_equal(np.array(shuffled, dtype=object), expected) + + def test_call_within_randomstate(self): + # Check that custom RandomState does not call into global state + m = random.RandomState() + res = np.array([0, 8, 7, 2, 1, 9, 4, 7, 0, 3]) + for i in range(3): + random.seed(i) + m.seed(4321) + # If m.state is not honored, the result will change + assert_array_equal(m.choice(10, size=10, p=np.ones(10)/10.), res) + + def test_multivariate_normal_size_types(self): + # Test for multivariate_normal issue with 'size' argument. + # Check that the multivariate_normal size argument can be a + # numpy integer. + random.multivariate_normal([0], [[0]], size=1) + random.multivariate_normal([0], [[0]], size=np.int_(1)) + random.multivariate_normal([0], [[0]], size=np.int64(1)) + + def test_beta_small_parameters(self): + # Test that beta with small a and b parameters does not produce + # NaNs due to roundoff errors causing 0 / 0, gh-5851 + random.seed(1234567890) + x = random.beta(0.0001, 0.0001, size=100) + assert_(not np.any(np.isnan(x)), 'Nans in random.beta') + + def test_choice_sum_of_probs_tolerance(self): + # The sum of probs should be 1.0 with some tolerance. + # For low precision dtypes the tolerance was too tight. + # See numpy github issue 6123. + random.seed(1234) + a = [1, 2, 3] + counts = [4, 4, 2] + for dt in np.float16, np.float32, np.float64: + probs = np.array(counts, dtype=dt) / sum(counts) + c = random.choice(a, p=probs) + assert_(c in a) + assert_raises(ValueError, random.choice, a, p=probs*0.9) + + def test_shuffle_of_array_of_different_length_strings(self): + # Test that permuting an array of different length strings + # will not cause a segfault on garbage collection + # Tests gh-7710 + random.seed(1234) + + a = np.array(['a', 'a' * 1000]) + + for _ in range(100): + random.shuffle(a) + + # Force Garbage Collection - should not segfault. + import gc + gc.collect() + + def test_shuffle_of_array_of_objects(self): + # Test that permuting an array of objects will not cause + # a segfault on garbage collection. + # See gh-7719 + random.seed(1234) + a = np.array([np.arange(1), np.arange(4)], dtype=object) + + for _ in range(1000): + random.shuffle(a) + + # Force Garbage Collection - should not segfault. + import gc + gc.collect() + + def test_permutation_subclass(self): + class N(np.ndarray): + pass + + random.seed(1) + orig = np.arange(3).view(N) + perm = random.permutation(orig) + assert_array_equal(perm, np.array([0, 2, 1])) + assert_array_equal(orig, np.arange(3).view(N)) + + class M: + a = np.arange(5) + + def __array__(self): + return self.a + + random.seed(1) + m = M() + perm = random.permutation(m) + assert_array_equal(perm, np.array([2, 1, 4, 0, 3])) + assert_array_equal(m.__array__(), np.arange(5)) + + def test_warns_byteorder(self): + # GH 13159 + other_byteord_dt = 'i4' + with pytest.deprecated_call(match='non-native byteorder is not'): + random.randint(0, 200, size=10, dtype=other_byteord_dt) + + def test_named_argument_initialization(self): + # GH 13669 + rs1 = np.random.RandomState(123456789) + rs2 = np.random.RandomState(seed=123456789) + assert rs1.randint(0, 100) == rs2.randint(0, 100) + + def test_choice_retun_dtype(self): + # GH 9867 + c = np.random.choice(10, p=[.1]*10, size=2) + assert c.dtype == np.dtype(int) + c = np.random.choice(10, p=[.1]*10, replace=False, size=2) + assert c.dtype == np.dtype(int) + c = np.random.choice(10, size=2) + assert c.dtype == np.dtype(int) + c = np.random.choice(10, replace=False, size=2) + assert c.dtype == np.dtype(int) + + @pytest.mark.skipif(np.iinfo('l').max < 2**32, + reason='Cannot test with 32-bit C long') + def test_randint_117(self): + # GH 14189 + random.seed(0) + expected = np.array([2357136044, 2546248239, 3071714933, 3626093760, + 2588848963, 3684848379, 2340255427, 3638918503, + 1819583497, 2678185683], dtype='int64') + actual = random.randint(2**32, size=10) + assert_array_equal(actual, expected) + + def test_p_zero_stream(self): + # Regression test for gh-14522. Ensure that future versions + # generate the same variates as version 1.16. + np.random.seed(12345) + assert_array_equal(random.binomial(1, [0, 0.25, 0.5, 0.75, 1]), + [0, 0, 0, 1, 1]) + + def test_n_zero_stream(self): + # Regression test for gh-14522. Ensure that future versions + # generate the same variates as version 1.16. + np.random.seed(8675309) + expected = np.array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [3, 4, 2, 3, 3, 1, 5, 3, 1, 3]]) + assert_array_equal(random.binomial([[0], [10]], 0.25, size=(2, 10)), + expected) + + +def test_multinomial_empty(): + # gh-20483 + # Ensure that empty p-vals are correctly handled + assert random.multinomial(10, []).shape == (0,) + assert random.multinomial(3, [], size=(7, 5, 3)).shape == (7, 5, 3, 0) + + +def test_multinomial_1d_pval(): + # gh-20483 + with pytest.raises(TypeError, match="pvals must be a 1-d"): + random.multinomial(10, 0.3) diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/test_regression.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/test_regression.py new file mode 100644 index 0000000000000000000000000000000000000000..8bf419875b3f37cd4cc121030d65be9fc77999a3 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/test_regression.py @@ -0,0 +1,149 @@ +import sys +from numpy.testing import ( + assert_, assert_array_equal, assert_raises, + ) +from numpy import random +import numpy as np + + +class TestRegression: + + def test_VonMises_range(self): + # Make sure generated random variables are in [-pi, pi]. + # Regression test for ticket #986. + for mu in np.linspace(-7., 7., 5): + r = random.mtrand.vonmises(mu, 1, 50) + assert_(np.all(r > -np.pi) and np.all(r <= np.pi)) + + def test_hypergeometric_range(self): + # Test for ticket #921 + assert_(np.all(np.random.hypergeometric(3, 18, 11, size=10) < 4)) + assert_(np.all(np.random.hypergeometric(18, 3, 11, size=10) > 0)) + + # Test for ticket #5623 + args = [ + (2**20 - 2, 2**20 - 2, 2**20 - 2), # Check for 32-bit systems + ] + is_64bits = sys.maxsize > 2**32 + if is_64bits and sys.platform != 'win32': + # Check for 64-bit systems + args.append((2**40 - 2, 2**40 - 2, 2**40 - 2)) + for arg in args: + assert_(np.random.hypergeometric(*arg) > 0) + + def test_logseries_convergence(self): + # Test for ticket #923 + N = 1000 + np.random.seed(0) + rvsn = np.random.logseries(0.8, size=N) + # these two frequency counts should be close to theoretical + # numbers with this large sample + # theoretical large N result is 0.49706795 + freq = np.sum(rvsn == 1) / N + msg = f'Frequency was {freq:f}, should be > 0.45' + assert_(freq > 0.45, msg) + # theoretical large N result is 0.19882718 + freq = np.sum(rvsn == 2) / N + msg = f'Frequency was {freq:f}, should be < 0.23' + assert_(freq < 0.23, msg) + + def test_shuffle_mixed_dimension(self): + # Test for trac ticket #2074 + for t in [[1, 2, 3, None], + [(1, 1), (2, 2), (3, 3), None], + [1, (2, 2), (3, 3), None], + [(1, 1), 2, 3, None]]: + np.random.seed(12345) + shuffled = list(t) + random.shuffle(shuffled) + expected = np.array([t[0], t[3], t[1], t[2]], dtype=object) + assert_array_equal(np.array(shuffled, dtype=object), expected) + + def test_call_within_randomstate(self): + # Check that custom RandomState does not call into global state + m = np.random.RandomState() + res = np.array([0, 8, 7, 2, 1, 9, 4, 7, 0, 3]) + for i in range(3): + np.random.seed(i) + m.seed(4321) + # If m.state is not honored, the result will change + assert_array_equal(m.choice(10, size=10, p=np.ones(10)/10.), res) + + def test_multivariate_normal_size_types(self): + # Test for multivariate_normal issue with 'size' argument. + # Check that the multivariate_normal size argument can be a + # numpy integer. + np.random.multivariate_normal([0], [[0]], size=1) + np.random.multivariate_normal([0], [[0]], size=np.int_(1)) + np.random.multivariate_normal([0], [[0]], size=np.int64(1)) + + def test_beta_small_parameters(self): + # Test that beta with small a and b parameters does not produce + # NaNs due to roundoff errors causing 0 / 0, gh-5851 + np.random.seed(1234567890) + x = np.random.beta(0.0001, 0.0001, size=100) + assert_(not np.any(np.isnan(x)), 'Nans in np.random.beta') + + def test_choice_sum_of_probs_tolerance(self): + # The sum of probs should be 1.0 with some tolerance. + # For low precision dtypes the tolerance was too tight. + # See numpy github issue 6123. + np.random.seed(1234) + a = [1, 2, 3] + counts = [4, 4, 2] + for dt in np.float16, np.float32, np.float64: + probs = np.array(counts, dtype=dt) / sum(counts) + c = np.random.choice(a, p=probs) + assert_(c in a) + assert_raises(ValueError, np.random.choice, a, p=probs*0.9) + + def test_shuffle_of_array_of_different_length_strings(self): + # Test that permuting an array of different length strings + # will not cause a segfault on garbage collection + # Tests gh-7710 + np.random.seed(1234) + + a = np.array(['a', 'a' * 1000]) + + for _ in range(100): + np.random.shuffle(a) + + # Force Garbage Collection - should not segfault. + import gc + gc.collect() + + def test_shuffle_of_array_of_objects(self): + # Test that permuting an array of objects will not cause + # a segfault on garbage collection. + # See gh-7719 + np.random.seed(1234) + a = np.array([np.arange(1), np.arange(4)], dtype=object) + + for _ in range(1000): + np.random.shuffle(a) + + # Force Garbage Collection - should not segfault. + import gc + gc.collect() + + def test_permutation_subclass(self): + class N(np.ndarray): + pass + + np.random.seed(1) + orig = np.arange(3).view(N) + perm = np.random.permutation(orig) + assert_array_equal(perm, np.array([0, 2, 1])) + assert_array_equal(orig, np.arange(3).view(N)) + + class M: + a = np.arange(5) + + def __array__(self): + return self.a + + np.random.seed(1) + m = M() + perm = np.random.permutation(m) + assert_array_equal(perm, np.array([2, 1, 4, 0, 3])) + assert_array_equal(m.__array__(), np.arange(5)) diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/test_seed_sequence.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/test_seed_sequence.py new file mode 100644 index 0000000000000000000000000000000000000000..f08cf80faafa2fc1a369eaf7dd4d6fcccd5e9158 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/test_seed_sequence.py @@ -0,0 +1,80 @@ +import numpy as np +from numpy.testing import assert_array_equal, assert_array_compare + +from numpy.random import SeedSequence + + +def test_reference_data(): + """ Check that SeedSequence generates data the same as the C++ reference. + + https://gist.github.com/imneme/540829265469e673d045 + """ + inputs = [ + [3735928559, 195939070, 229505742, 305419896], + [3668361503, 4165561550, 1661411377, 3634257570], + [164546577, 4166754639, 1765190214, 1303880213], + [446610472, 3941463886, 522937693, 1882353782], + [1864922766, 1719732118, 3882010307, 1776744564], + [4141682960, 3310988675, 553637289, 902896340], + [1134851934, 2352871630, 3699409824, 2648159817], + [1240956131, 3107113773, 1283198141, 1924506131], + [2669565031, 579818610, 3042504477, 2774880435], + [2766103236, 2883057919, 4029656435, 862374500], + ] + outputs = [ + [3914649087, 576849849, 3593928901, 2229911004], + [2240804226, 3691353228, 1365957195, 2654016646], + [3562296087, 3191708229, 1147942216, 3726991905], + [1403443605, 3591372999, 1291086759, 441919183], + [1086200464, 2191331643, 560336446, 3658716651], + [3249937430, 2346751812, 847844327, 2996632307], + [2584285912, 4034195531, 3523502488, 169742686], + [959045797, 3875435559, 1886309314, 359682705], + [3978441347, 432478529, 3223635119, 138903045], + [296367413, 4262059219, 13109864, 3283683422], + ] + outputs64 = [ + [2477551240072187391, 9577394838764454085], + [15854241394484835714, 11398914698975566411], + [13708282465491374871, 16007308345579681096], + [15424829579845884309, 1898028439751125927], + [9411697742461147792, 15714068361935982142], + [10079222287618677782, 12870437757549876199], + [17326737873898640088, 729039288628699544], + [16644868984619524261, 1544825456798124994], + [1857481142255628931, 596584038813451439], + [18305404959516669237, 14103312907920476776], + ] + for seed, expected, expected64 in zip(inputs, outputs, outputs64): + expected = np.array(expected, dtype=np.uint32) + ss = SeedSequence(seed) + state = ss.generate_state(len(expected)) + assert_array_equal(state, expected) + state64 = ss.generate_state(len(expected64), dtype=np.uint64) + assert_array_equal(state64, expected64) + + +def test_zero_padding(): + """ Ensure that the implicit zero-padding does not cause problems. + """ + # Ensure that large integers are inserted in little-endian fashion to avoid + # trailing 0s. + ss0 = SeedSequence(42) + ss1 = SeedSequence(42 << 32) + assert_array_compare( + np.not_equal, + ss0.generate_state(4), + ss1.generate_state(4)) + + # Ensure backwards compatibility with the original 0.17 release for small + # integers and no spawn key. + expected42 = np.array([3444837047, 2669555309, 2046530742, 3581440988], + dtype=np.uint32) + assert_array_equal(SeedSequence(42).generate_state(4), expected42) + + # Regression test for gh-16539 to ensure that the implicit 0s don't + # conflict with spawn keys. + assert_array_compare( + np.not_equal, + SeedSequence(42, spawn_key=(0,)).generate_state(4), + expected42) diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/test_smoke.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/test_smoke.py new file mode 100644 index 0000000000000000000000000000000000000000..9becc434d0d1a66b7c9987d8c5dffdf221fd45b1 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/tests/test_smoke.py @@ -0,0 +1,818 @@ +import pickle +from functools import partial + +import numpy as np +import pytest +from numpy.testing import assert_equal, assert_, assert_array_equal +from numpy.random import (Generator, MT19937, PCG64, PCG64DXSM, Philox, SFC64) + +@pytest.fixture(scope='module', + params=(np.bool_, np.int8, np.int16, np.int32, np.int64, + np.uint8, np.uint16, np.uint32, np.uint64)) +def dtype(request): + return request.param + + +def params_0(f): + val = f() + assert_(np.isscalar(val)) + val = f(10) + assert_(val.shape == (10,)) + val = f((10, 10)) + assert_(val.shape == (10, 10)) + val = f((10, 10, 10)) + assert_(val.shape == (10, 10, 10)) + val = f(size=(5, 5)) + assert_(val.shape == (5, 5)) + + +def params_1(f, bounded=False): + a = 5.0 + b = np.arange(2.0, 12.0) + c = np.arange(2.0, 102.0).reshape((10, 10)) + d = np.arange(2.0, 1002.0).reshape((10, 10, 10)) + e = np.array([2.0, 3.0]) + g = np.arange(2.0, 12.0).reshape((1, 10, 1)) + if bounded: + a = 0.5 + b = b / (1.5 * b.max()) + c = c / (1.5 * c.max()) + d = d / (1.5 * d.max()) + e = e / (1.5 * e.max()) + g = g / (1.5 * g.max()) + + # Scalar + f(a) + # Scalar - size + f(a, size=(10, 10)) + # 1d + f(b) + # 2d + f(c) + # 3d + f(d) + # 1d size + f(b, size=10) + # 2d - size - broadcast + f(e, size=(10, 2)) + # 3d - size + f(g, size=(10, 10, 10)) + + +def comp_state(state1, state2): + identical = True + if isinstance(state1, dict): + for key in state1: + identical &= comp_state(state1[key], state2[key]) + elif type(state1) != type(state2): + identical &= type(state1) == type(state2) + else: + if (isinstance(state1, (list, tuple, np.ndarray)) and isinstance( + state2, (list, tuple, np.ndarray))): + for s1, s2 in zip(state1, state2): + identical &= comp_state(s1, s2) + else: + identical &= state1 == state2 + return identical + + +def warmup(rg, n=None): + if n is None: + n = 11 + np.random.randint(0, 20) + rg.standard_normal(n) + rg.standard_normal(n) + rg.standard_normal(n, dtype=np.float32) + rg.standard_normal(n, dtype=np.float32) + rg.integers(0, 2 ** 24, n, dtype=np.uint64) + rg.integers(0, 2 ** 48, n, dtype=np.uint64) + rg.standard_gamma(11.0, n) + rg.standard_gamma(11.0, n, dtype=np.float32) + rg.random(n, dtype=np.float64) + rg.random(n, dtype=np.float32) + + +class RNG: + @classmethod + def setup_class(cls): + # Overridden in test classes. Place holder to silence IDE noise + cls.bit_generator = PCG64 + cls.advance = None + cls.seed = [12345] + cls.rg = Generator(cls.bit_generator(*cls.seed)) + cls.initial_state = cls.rg.bit_generator.state + cls.seed_vector_bits = 64 + cls._extra_setup() + + @classmethod + def _extra_setup(cls): + cls.vec_1d = np.arange(2.0, 102.0) + cls.vec_2d = np.arange(2.0, 102.0)[None, :] + cls.mat = np.arange(2.0, 102.0, 0.01).reshape((100, 100)) + cls.seed_error = TypeError + + def _reset_state(self): + self.rg.bit_generator.state = self.initial_state + + def test_init(self): + rg = Generator(self.bit_generator()) + state = rg.bit_generator.state + rg.standard_normal(1) + rg.standard_normal(1) + rg.bit_generator.state = state + new_state = rg.bit_generator.state + assert_(comp_state(state, new_state)) + + def test_advance(self): + state = self.rg.bit_generator.state + if hasattr(self.rg.bit_generator, 'advance'): + self.rg.bit_generator.advance(self.advance) + assert_(not comp_state(state, self.rg.bit_generator.state)) + else: + bitgen_name = self.rg.bit_generator.__class__.__name__ + pytest.skip(f'Advance is not supported by {bitgen_name}') + + def test_jump(self): + state = self.rg.bit_generator.state + if hasattr(self.rg.bit_generator, 'jumped'): + bit_gen2 = self.rg.bit_generator.jumped() + jumped_state = bit_gen2.state + assert_(not comp_state(state, jumped_state)) + self.rg.random(2 * 3 * 5 * 7 * 11 * 13 * 17) + self.rg.bit_generator.state = state + bit_gen3 = self.rg.bit_generator.jumped() + rejumped_state = bit_gen3.state + assert_(comp_state(jumped_state, rejumped_state)) + else: + bitgen_name = self.rg.bit_generator.__class__.__name__ + if bitgen_name not in ('SFC64',): + raise AttributeError(f'no "jumped" in {bitgen_name}') + pytest.skip(f'Jump is not supported by {bitgen_name}') + + def test_uniform(self): + r = self.rg.uniform(-1.0, 0.0, size=10) + assert_(len(r) == 10) + assert_((r > -1).all()) + assert_((r <= 0).all()) + + def test_uniform_array(self): + r = self.rg.uniform(np.array([-1.0] * 10), 0.0, size=10) + assert_(len(r) == 10) + assert_((r > -1).all()) + assert_((r <= 0).all()) + r = self.rg.uniform(np.array([-1.0] * 10), + np.array([0.0] * 10), size=10) + assert_(len(r) == 10) + assert_((r > -1).all()) + assert_((r <= 0).all()) + r = self.rg.uniform(-1.0, np.array([0.0] * 10), size=10) + assert_(len(r) == 10) + assert_((r > -1).all()) + assert_((r <= 0).all()) + + def test_random(self): + assert_(len(self.rg.random(10)) == 10) + params_0(self.rg.random) + + def test_standard_normal_zig(self): + assert_(len(self.rg.standard_normal(10)) == 10) + + def test_standard_normal(self): + assert_(len(self.rg.standard_normal(10)) == 10) + params_0(self.rg.standard_normal) + + def test_standard_gamma(self): + assert_(len(self.rg.standard_gamma(10, 10)) == 10) + assert_(len(self.rg.standard_gamma(np.array([10] * 10), 10)) == 10) + params_1(self.rg.standard_gamma) + + def test_standard_exponential(self): + assert_(len(self.rg.standard_exponential(10)) == 10) + params_0(self.rg.standard_exponential) + + def test_standard_exponential_float(self): + randoms = self.rg.standard_exponential(10, dtype='float32') + assert_(len(randoms) == 10) + assert randoms.dtype == np.float32 + params_0(partial(self.rg.standard_exponential, dtype='float32')) + + def test_standard_exponential_float_log(self): + randoms = self.rg.standard_exponential(10, dtype='float32', + method='inv') + assert_(len(randoms) == 10) + assert randoms.dtype == np.float32 + params_0(partial(self.rg.standard_exponential, dtype='float32', + method='inv')) + + def test_standard_cauchy(self): + assert_(len(self.rg.standard_cauchy(10)) == 10) + params_0(self.rg.standard_cauchy) + + def test_standard_t(self): + assert_(len(self.rg.standard_t(10, 10)) == 10) + params_1(self.rg.standard_t) + + def test_binomial(self): + assert_(self.rg.binomial(10, .5) >= 0) + assert_(self.rg.binomial(1000, .5) >= 0) + + def test_reset_state(self): + state = self.rg.bit_generator.state + int_1 = self.rg.integers(2**31) + self.rg.bit_generator.state = state + int_2 = self.rg.integers(2**31) + assert_(int_1 == int_2) + + def test_entropy_init(self): + rg = Generator(self.bit_generator()) + rg2 = Generator(self.bit_generator()) + assert_(not comp_state(rg.bit_generator.state, + rg2.bit_generator.state)) + + def test_seed(self): + rg = Generator(self.bit_generator(*self.seed)) + rg2 = Generator(self.bit_generator(*self.seed)) + rg.random() + rg2.random() + assert_(comp_state(rg.bit_generator.state, rg2.bit_generator.state)) + + def test_reset_state_gauss(self): + rg = Generator(self.bit_generator(*self.seed)) + rg.standard_normal() + state = rg.bit_generator.state + n1 = rg.standard_normal(size=10) + rg2 = Generator(self.bit_generator()) + rg2.bit_generator.state = state + n2 = rg2.standard_normal(size=10) + assert_array_equal(n1, n2) + + def test_reset_state_uint32(self): + rg = Generator(self.bit_generator(*self.seed)) + rg.integers(0, 2 ** 24, 120, dtype=np.uint32) + state = rg.bit_generator.state + n1 = rg.integers(0, 2 ** 24, 10, dtype=np.uint32) + rg2 = Generator(self.bit_generator()) + rg2.bit_generator.state = state + n2 = rg2.integers(0, 2 ** 24, 10, dtype=np.uint32) + assert_array_equal(n1, n2) + + def test_reset_state_float(self): + rg = Generator(self.bit_generator(*self.seed)) + rg.random(dtype='float32') + state = rg.bit_generator.state + n1 = rg.random(size=10, dtype='float32') + rg2 = Generator(self.bit_generator()) + rg2.bit_generator.state = state + n2 = rg2.random(size=10, dtype='float32') + assert_((n1 == n2).all()) + + def test_shuffle(self): + original = np.arange(200, 0, -1) + permuted = self.rg.permutation(original) + assert_((original != permuted).any()) + + def test_permutation(self): + original = np.arange(200, 0, -1) + permuted = self.rg.permutation(original) + assert_((original != permuted).any()) + + def test_beta(self): + vals = self.rg.beta(2.0, 2.0, 10) + assert_(len(vals) == 10) + vals = self.rg.beta(np.array([2.0] * 10), 2.0) + assert_(len(vals) == 10) + vals = self.rg.beta(2.0, np.array([2.0] * 10)) + assert_(len(vals) == 10) + vals = self.rg.beta(np.array([2.0] * 10), np.array([2.0] * 10)) + assert_(len(vals) == 10) + vals = self.rg.beta(np.array([2.0] * 10), np.array([[2.0]] * 10)) + assert_(vals.shape == (10, 10)) + + def test_bytes(self): + vals = self.rg.bytes(10) + assert_(len(vals) == 10) + + def test_chisquare(self): + vals = self.rg.chisquare(2.0, 10) + assert_(len(vals) == 10) + params_1(self.rg.chisquare) + + def test_exponential(self): + vals = self.rg.exponential(2.0, 10) + assert_(len(vals) == 10) + params_1(self.rg.exponential) + + def test_f(self): + vals = self.rg.f(3, 1000, 10) + assert_(len(vals) == 10) + + def test_gamma(self): + vals = self.rg.gamma(3, 2, 10) + assert_(len(vals) == 10) + + def test_geometric(self): + vals = self.rg.geometric(0.5, 10) + assert_(len(vals) == 10) + params_1(self.rg.exponential, bounded=True) + + def test_gumbel(self): + vals = self.rg.gumbel(2.0, 2.0, 10) + assert_(len(vals) == 10) + + def test_laplace(self): + vals = self.rg.laplace(2.0, 2.0, 10) + assert_(len(vals) == 10) + + def test_logitic(self): + vals = self.rg.logistic(2.0, 2.0, 10) + assert_(len(vals) == 10) + + def test_logseries(self): + vals = self.rg.logseries(0.5, 10) + assert_(len(vals) == 10) + + def test_negative_binomial(self): + vals = self.rg.negative_binomial(10, 0.2, 10) + assert_(len(vals) == 10) + + def test_noncentral_chisquare(self): + vals = self.rg.noncentral_chisquare(10, 2, 10) + assert_(len(vals) == 10) + + def test_noncentral_f(self): + vals = self.rg.noncentral_f(3, 1000, 2, 10) + assert_(len(vals) == 10) + vals = self.rg.noncentral_f(np.array([3] * 10), 1000, 2) + assert_(len(vals) == 10) + vals = self.rg.noncentral_f(3, np.array([1000] * 10), 2) + assert_(len(vals) == 10) + vals = self.rg.noncentral_f(3, 1000, np.array([2] * 10)) + assert_(len(vals) == 10) + + def test_normal(self): + vals = self.rg.normal(10, 0.2, 10) + assert_(len(vals) == 10) + + def test_pareto(self): + vals = self.rg.pareto(3.0, 10) + assert_(len(vals) == 10) + + def test_poisson(self): + vals = self.rg.poisson(10, 10) + assert_(len(vals) == 10) + vals = self.rg.poisson(np.array([10] * 10)) + assert_(len(vals) == 10) + params_1(self.rg.poisson) + + def test_power(self): + vals = self.rg.power(0.2, 10) + assert_(len(vals) == 10) + + def test_integers(self): + vals = self.rg.integers(10, 20, 10) + assert_(len(vals) == 10) + + def test_rayleigh(self): + vals = self.rg.rayleigh(0.2, 10) + assert_(len(vals) == 10) + params_1(self.rg.rayleigh, bounded=True) + + def test_vonmises(self): + vals = self.rg.vonmises(10, 0.2, 10) + assert_(len(vals) == 10) + + def test_wald(self): + vals = self.rg.wald(1.0, 1.0, 10) + assert_(len(vals) == 10) + + def test_weibull(self): + vals = self.rg.weibull(1.0, 10) + assert_(len(vals) == 10) + + def test_zipf(self): + vals = self.rg.zipf(10, 10) + assert_(len(vals) == 10) + vals = self.rg.zipf(self.vec_1d) + assert_(len(vals) == 100) + vals = self.rg.zipf(self.vec_2d) + assert_(vals.shape == (1, 100)) + vals = self.rg.zipf(self.mat) + assert_(vals.shape == (100, 100)) + + def test_hypergeometric(self): + vals = self.rg.hypergeometric(25, 25, 20) + assert_(np.isscalar(vals)) + vals = self.rg.hypergeometric(np.array([25] * 10), 25, 20) + assert_(vals.shape == (10,)) + + def test_triangular(self): + vals = self.rg.triangular(-5, 0, 5) + assert_(np.isscalar(vals)) + vals = self.rg.triangular(-5, np.array([0] * 10), 5) + assert_(vals.shape == (10,)) + + def test_multivariate_normal(self): + mean = [0, 0] + cov = [[1, 0], [0, 100]] # diagonal covariance + x = self.rg.multivariate_normal(mean, cov, 5000) + assert_(x.shape == (5000, 2)) + x_zig = self.rg.multivariate_normal(mean, cov, 5000) + assert_(x.shape == (5000, 2)) + x_inv = self.rg.multivariate_normal(mean, cov, 5000) + assert_(x.shape == (5000, 2)) + assert_((x_zig != x_inv).any()) + + def test_multinomial(self): + vals = self.rg.multinomial(100, [1.0 / 3, 2.0 / 3]) + assert_(vals.shape == (2,)) + vals = self.rg.multinomial(100, [1.0 / 3, 2.0 / 3], size=10) + assert_(vals.shape == (10, 2)) + + def test_dirichlet(self): + s = self.rg.dirichlet((10, 5, 3), 20) + assert_(s.shape == (20, 3)) + + def test_pickle(self): + pick = pickle.dumps(self.rg) + unpick = pickle.loads(pick) + assert_((type(self.rg) == type(unpick))) + assert_(comp_state(self.rg.bit_generator.state, + unpick.bit_generator.state)) + + pick = pickle.dumps(self.rg) + unpick = pickle.loads(pick) + assert_((type(self.rg) == type(unpick))) + assert_(comp_state(self.rg.bit_generator.state, + unpick.bit_generator.state)) + + def test_seed_array(self): + if self.seed_vector_bits is None: + bitgen_name = self.bit_generator.__name__ + pytest.skip(f'Vector seeding is not supported by {bitgen_name}') + + if self.seed_vector_bits == 32: + dtype = np.uint32 + else: + dtype = np.uint64 + seed = np.array([1], dtype=dtype) + bg = self.bit_generator(seed) + state1 = bg.state + bg = self.bit_generator(1) + state2 = bg.state + assert_(comp_state(state1, state2)) + + seed = np.arange(4, dtype=dtype) + bg = self.bit_generator(seed) + state1 = bg.state + bg = self.bit_generator(seed[0]) + state2 = bg.state + assert_(not comp_state(state1, state2)) + + seed = np.arange(1500, dtype=dtype) + bg = self.bit_generator(seed) + state1 = bg.state + bg = self.bit_generator(seed[0]) + state2 = bg.state + assert_(not comp_state(state1, state2)) + + seed = 2 ** np.mod(np.arange(1500, dtype=dtype), + self.seed_vector_bits - 1) + 1 + bg = self.bit_generator(seed) + state1 = bg.state + bg = self.bit_generator(seed[0]) + state2 = bg.state + assert_(not comp_state(state1, state2)) + + def test_uniform_float(self): + rg = Generator(self.bit_generator(12345)) + warmup(rg) + state = rg.bit_generator.state + r1 = rg.random(11, dtype=np.float32) + rg2 = Generator(self.bit_generator()) + warmup(rg2) + rg2.bit_generator.state = state + r2 = rg2.random(11, dtype=np.float32) + assert_array_equal(r1, r2) + assert_equal(r1.dtype, np.float32) + assert_(comp_state(rg.bit_generator.state, rg2.bit_generator.state)) + + def test_gamma_floats(self): + rg = Generator(self.bit_generator()) + warmup(rg) + state = rg.bit_generator.state + r1 = rg.standard_gamma(4.0, 11, dtype=np.float32) + rg2 = Generator(self.bit_generator()) + warmup(rg2) + rg2.bit_generator.state = state + r2 = rg2.standard_gamma(4.0, 11, dtype=np.float32) + assert_array_equal(r1, r2) + assert_equal(r1.dtype, np.float32) + assert_(comp_state(rg.bit_generator.state, rg2.bit_generator.state)) + + def test_normal_floats(self): + rg = Generator(self.bit_generator()) + warmup(rg) + state = rg.bit_generator.state + r1 = rg.standard_normal(11, dtype=np.float32) + rg2 = Generator(self.bit_generator()) + warmup(rg2) + rg2.bit_generator.state = state + r2 = rg2.standard_normal(11, dtype=np.float32) + assert_array_equal(r1, r2) + assert_equal(r1.dtype, np.float32) + assert_(comp_state(rg.bit_generator.state, rg2.bit_generator.state)) + + def test_normal_zig_floats(self): + rg = Generator(self.bit_generator()) + warmup(rg) + state = rg.bit_generator.state + r1 = rg.standard_normal(11, dtype=np.float32) + rg2 = Generator(self.bit_generator()) + warmup(rg2) + rg2.bit_generator.state = state + r2 = rg2.standard_normal(11, dtype=np.float32) + assert_array_equal(r1, r2) + assert_equal(r1.dtype, np.float32) + assert_(comp_state(rg.bit_generator.state, rg2.bit_generator.state)) + + def test_output_fill(self): + rg = self.rg + state = rg.bit_generator.state + size = (31, 7, 97) + existing = np.empty(size) + rg.bit_generator.state = state + rg.standard_normal(out=existing) + rg.bit_generator.state = state + direct = rg.standard_normal(size=size) + assert_equal(direct, existing) + + sized = np.empty(size) + rg.bit_generator.state = state + rg.standard_normal(out=sized, size=sized.shape) + + existing = np.empty(size, dtype=np.float32) + rg.bit_generator.state = state + rg.standard_normal(out=existing, dtype=np.float32) + rg.bit_generator.state = state + direct = rg.standard_normal(size=size, dtype=np.float32) + assert_equal(direct, existing) + + def test_output_filling_uniform(self): + rg = self.rg + state = rg.bit_generator.state + size = (31, 7, 97) + existing = np.empty(size) + rg.bit_generator.state = state + rg.random(out=existing) + rg.bit_generator.state = state + direct = rg.random(size=size) + assert_equal(direct, existing) + + existing = np.empty(size, dtype=np.float32) + rg.bit_generator.state = state + rg.random(out=existing, dtype=np.float32) + rg.bit_generator.state = state + direct = rg.random(size=size, dtype=np.float32) + assert_equal(direct, existing) + + def test_output_filling_exponential(self): + rg = self.rg + state = rg.bit_generator.state + size = (31, 7, 97) + existing = np.empty(size) + rg.bit_generator.state = state + rg.standard_exponential(out=existing) + rg.bit_generator.state = state + direct = rg.standard_exponential(size=size) + assert_equal(direct, existing) + + existing = np.empty(size, dtype=np.float32) + rg.bit_generator.state = state + rg.standard_exponential(out=existing, dtype=np.float32) + rg.bit_generator.state = state + direct = rg.standard_exponential(size=size, dtype=np.float32) + assert_equal(direct, existing) + + def test_output_filling_gamma(self): + rg = self.rg + state = rg.bit_generator.state + size = (31, 7, 97) + existing = np.zeros(size) + rg.bit_generator.state = state + rg.standard_gamma(1.0, out=existing) + rg.bit_generator.state = state + direct = rg.standard_gamma(1.0, size=size) + assert_equal(direct, existing) + + existing = np.zeros(size, dtype=np.float32) + rg.bit_generator.state = state + rg.standard_gamma(1.0, out=existing, dtype=np.float32) + rg.bit_generator.state = state + direct = rg.standard_gamma(1.0, size=size, dtype=np.float32) + assert_equal(direct, existing) + + def test_output_filling_gamma_broadcast(self): + rg = self.rg + state = rg.bit_generator.state + size = (31, 7, 97) + mu = np.arange(97.0) + 1.0 + existing = np.zeros(size) + rg.bit_generator.state = state + rg.standard_gamma(mu, out=existing) + rg.bit_generator.state = state + direct = rg.standard_gamma(mu, size=size) + assert_equal(direct, existing) + + existing = np.zeros(size, dtype=np.float32) + rg.bit_generator.state = state + rg.standard_gamma(mu, out=existing, dtype=np.float32) + rg.bit_generator.state = state + direct = rg.standard_gamma(mu, size=size, dtype=np.float32) + assert_equal(direct, existing) + + def test_output_fill_error(self): + rg = self.rg + size = (31, 7, 97) + existing = np.empty(size) + with pytest.raises(TypeError): + rg.standard_normal(out=existing, dtype=np.float32) + with pytest.raises(ValueError): + rg.standard_normal(out=existing[::3]) + existing = np.empty(size, dtype=np.float32) + with pytest.raises(TypeError): + rg.standard_normal(out=existing, dtype=np.float64) + + existing = np.zeros(size, dtype=np.float32) + with pytest.raises(TypeError): + rg.standard_gamma(1.0, out=existing, dtype=np.float64) + with pytest.raises(ValueError): + rg.standard_gamma(1.0, out=existing[::3], dtype=np.float32) + existing = np.zeros(size, dtype=np.float64) + with pytest.raises(TypeError): + rg.standard_gamma(1.0, out=existing, dtype=np.float32) + with pytest.raises(ValueError): + rg.standard_gamma(1.0, out=existing[::3]) + + def test_integers_broadcast(self, dtype): + if dtype == np.bool_: + upper = 2 + lower = 0 + else: + info = np.iinfo(dtype) + upper = int(info.max) + 1 + lower = info.min + self._reset_state() + a = self.rg.integers(lower, [upper] * 10, dtype=dtype) + self._reset_state() + b = self.rg.integers([lower] * 10, upper, dtype=dtype) + assert_equal(a, b) + self._reset_state() + c = self.rg.integers(lower, upper, size=10, dtype=dtype) + assert_equal(a, c) + self._reset_state() + d = self.rg.integers(np.array( + [lower] * 10), np.array([upper], dtype=object), size=10, + dtype=dtype) + assert_equal(a, d) + self._reset_state() + e = self.rg.integers( + np.array([lower] * 10), np.array([upper] * 10), size=10, + dtype=dtype) + assert_equal(a, e) + + self._reset_state() + a = self.rg.integers(0, upper, size=10, dtype=dtype) + self._reset_state() + b = self.rg.integers([upper] * 10, dtype=dtype) + assert_equal(a, b) + + def test_integers_numpy(self, dtype): + high = np.array([1]) + low = np.array([0]) + + out = self.rg.integers(low, high, dtype=dtype) + assert out.shape == (1,) + + out = self.rg.integers(low[0], high, dtype=dtype) + assert out.shape == (1,) + + out = self.rg.integers(low, high[0], dtype=dtype) + assert out.shape == (1,) + + def test_integers_broadcast_errors(self, dtype): + if dtype == np.bool_: + upper = 2 + lower = 0 + else: + info = np.iinfo(dtype) + upper = int(info.max) + 1 + lower = info.min + with pytest.raises(ValueError): + self.rg.integers(lower, [upper + 1] * 10, dtype=dtype) + with pytest.raises(ValueError): + self.rg.integers(lower - 1, [upper] * 10, dtype=dtype) + with pytest.raises(ValueError): + self.rg.integers([lower - 1], [upper] * 10, dtype=dtype) + with pytest.raises(ValueError): + self.rg.integers([0], [0], dtype=dtype) + + +class TestMT19937(RNG): + @classmethod + def setup_class(cls): + cls.bit_generator = MT19937 + cls.advance = None + cls.seed = [2 ** 21 + 2 ** 16 + 2 ** 5 + 1] + cls.rg = Generator(cls.bit_generator(*cls.seed)) + cls.initial_state = cls.rg.bit_generator.state + cls.seed_vector_bits = 32 + cls._extra_setup() + cls.seed_error = ValueError + + def test_numpy_state(self): + nprg = np.random.RandomState() + nprg.standard_normal(99) + state = nprg.get_state() + self.rg.bit_generator.state = state + state2 = self.rg.bit_generator.state + assert_((state[1] == state2['state']['key']).all()) + assert_((state[2] == state2['state']['pos'])) + + +class TestPhilox(RNG): + @classmethod + def setup_class(cls): + cls.bit_generator = Philox + cls.advance = 2**63 + 2**31 + 2**15 + 1 + cls.seed = [12345] + cls.rg = Generator(cls.bit_generator(*cls.seed)) + cls.initial_state = cls.rg.bit_generator.state + cls.seed_vector_bits = 64 + cls._extra_setup() + + +class TestSFC64(RNG): + @classmethod + def setup_class(cls): + cls.bit_generator = SFC64 + cls.advance = None + cls.seed = [12345] + cls.rg = Generator(cls.bit_generator(*cls.seed)) + cls.initial_state = cls.rg.bit_generator.state + cls.seed_vector_bits = 192 + cls._extra_setup() + + +class TestPCG64(RNG): + @classmethod + def setup_class(cls): + cls.bit_generator = PCG64 + cls.advance = 2**63 + 2**31 + 2**15 + 1 + cls.seed = [12345] + cls.rg = Generator(cls.bit_generator(*cls.seed)) + cls.initial_state = cls.rg.bit_generator.state + cls.seed_vector_bits = 64 + cls._extra_setup() + + +class TestPCG64DXSM(RNG): + @classmethod + def setup_class(cls): + cls.bit_generator = PCG64DXSM + cls.advance = 2**63 + 2**31 + 2**15 + 1 + cls.seed = [12345] + cls.rg = Generator(cls.bit_generator(*cls.seed)) + cls.initial_state = cls.rg.bit_generator.state + cls.seed_vector_bits = 64 + cls._extra_setup() + + +class TestDefaultRNG(RNG): + @classmethod + def setup_class(cls): + # This will duplicate some tests that directly instantiate a fresh + # Generator(), but that's okay. + cls.bit_generator = PCG64 + cls.advance = 2**63 + 2**31 + 2**15 + 1 + cls.seed = [12345] + cls.rg = np.random.default_rng(*cls.seed) + cls.initial_state = cls.rg.bit_generator.state + cls.seed_vector_bits = 64 + cls._extra_setup() + + def test_default_is_pcg64(self): + # In order to change the default BitGenerator, we'll go through + # a deprecation cycle to move to a different function. + assert_(isinstance(self.rg.bit_generator, PCG64)) + + def test_seed(self): + np.random.default_rng() + np.random.default_rng(None) + np.random.default_rng(12345) + np.random.default_rng(0) + np.random.default_rng(43660444402423911716352051725018508569) + np.random.default_rng([43660444402423911716352051725018508569, + 279705150948142787361475340226491943209]) + with pytest.raises(ValueError): + np.random.default_rng(-1) + with pytest.raises(ValueError): + np.random.default_rng([12345, -1]) diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/logs/infer_not5_bottleneck128_170k_decode32_ema_20260611/lr3e3.log b/LTA_openwebtext_dualt/mini_owt_logdirichlet/logs/infer_not5_bottleneck128_170k_decode32_ema_20260611/lr3e3.log new file mode 100644 index 0000000000000000000000000000000000000000..e9ef8d9ed7d233321b908409d71ead83b7424687 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/logs/infer_not5_bottleneck128_170k_decode32_ema_20260611/lr3e3.log @@ -0,0 +1,29 @@ +checkpoint=runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_2x8gpu_50ep_lr3e3_ema0p9999_elfopt_not5_bottleneck128_unfixed_norm_stateprobadd_selfcond_ce_fast_20260609_230847/step_170000.pt +use_ema=1 +step=170000 +decode_steps=32 +n=64 chunk_n=8 gpu=2 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260611 +[2026-06-11T21:37:16+00:00] infer step=170000 decode=32 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260611/owt_t5_not5_bottleneck128_norm_stateprobadd_selfcond_ce_fast_lr3e3_ema0p9999_step170000_ema_sc1p0_decode32_n64 +[2026-06-11T21:37:16+00:00] run decode=32 chunk=0 n=8 seed=123 +[2026-06-11T21:37:22+00:00] done decode=32 chunk=0 +[2026-06-11T21:37:22+00:00] run decode=32 chunk=1 n=8 seed=124 +[2026-06-11T21:37:29+00:00] done decode=32 chunk=1 +[2026-06-11T21:37:29+00:00] run decode=32 chunk=2 n=8 seed=125 +[2026-06-11T21:37:36+00:00] done decode=32 chunk=2 +[2026-06-11T21:37:36+00:00] run decode=32 chunk=3 n=8 seed=126 +[2026-06-11T21:37:43+00:00] done decode=32 chunk=3 +[2026-06-11T21:37:43+00:00] run decode=32 chunk=4 n=8 seed=127 +[2026-06-11T21:37:50+00:00] done decode=32 chunk=4 +[2026-06-11T21:37:50+00:00] run decode=32 chunk=5 n=8 seed=128 +[2026-06-11T21:37:57+00:00] done decode=32 chunk=5 +[2026-06-11T21:37:57+00:00] run decode=32 chunk=6 n=8 seed=129 +[2026-06-11T21:38:04+00:00] done decode=32 chunk=6 +[2026-06-11T21:38:04+00:00] run decode=32 chunk=7 n=8 seed=130 +[2026-06-11T21:38:11+00:00] done decode=32 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260611/owt_t5_not5_bottleneck128_norm_stateprobadd_selfcond_ce_fast_lr3e3_ema0p9999_step170000_ema_sc1p0_decode32_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 32.86594428444645 5.137365192808393 0.09408659895050174 0.5231767756586316 0.025577684352655967 62 63 60706 65174 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260611/owt_t5_not5_bottleneck128_norm_stateprobadd_selfcond_ce_fast_lr3e3_ema0p9999_step170000_ema_sc1p0_decode32_n64/sc1p0 +sc1p0 pre_eos 37.55003027517274 5.1556471729724 0.09639842165663172 0.5360006288319447 0.026206159312068666 0 0 57550 63611 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260611/owt_t5_not5_bottleneck128_norm_stateprobadd_selfcond_ce_fast_lr3e3_ema0p9999_step170000_ema_sc1p0_decode32_n64/sc1p0 +[2026-06-11T21:38:24+00:00] done