diff --git a/wandb/run-20241030_010759-imp8g625/files/config.yaml b/wandb/run-20241030_010759-imp8g625/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f8546482f67cbb9041af9d756594322c485d67a9 --- /dev/null +++ b/wandb/run-20241030_010759-imp8g625/files/config.yaml @@ -0,0 +1,47 @@ +_wandb: + value: + cli_version: 0.18.5 + m: [] + python_version: 3.9.19 + t: + "1": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "2": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "3": + - 13 + - 23 + - 55 + "4": 3.9.19 + "5": 0.18.5 + "6": 4.45.1 + "8": + - 5 + "12": 0.18.5 + "13": linux-x86_64 +batch_size: + value: 3 +epoch: + value: 7 +perturbation: + value: reverse_control +seed: + value: 0 +train_set: + value: 10M diff --git a/wandb/run-20241030_010759-imp8g625/files/output.log b/wandb/run-20241030_010759-imp8g625/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..1777f063b107f651dddc063d1d3a3fa80015bf9e --- /dev/null +++ b/wandb/run-20241030_010759-imp8g625/files/output.log @@ -0,0 +1,4 @@ +Traceback (most recent call last): + File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 162, in + dataset_name = f"babylm_{args.perturbation}_{args.train_zset}_seed{args.seed}" +AttributeError: 'Namespace' object has no attribute 'train_zset' diff --git a/wandb/run-20241030_010759-imp8g625/run-imp8g625.wandb b/wandb/run-20241030_010759-imp8g625/run-imp8g625.wandb new file mode 100644 index 0000000000000000000000000000000000000000..0dccba575debeb9c5f8dbe14c509670d62660491 Binary files /dev/null and b/wandb/run-20241030_010759-imp8g625/run-imp8g625.wandb differ diff --git a/wandb/run-20241030_013141-bkcoggdw/logs/debug.log b/wandb/run-20241030_013141-bkcoggdw/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..1ed1dc93a3636ee177ec4f3cc79ab1f5970a31dd --- /dev/null +++ b/wandb/run-20241030_013141-bkcoggdw/logs/debug.log @@ -0,0 +1,27 @@ +2024-10-30 01:31:41,691 INFO MainThread:335755 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-10-30 01:31:41,691 INFO MainThread:335755 [wandb_setup.py:_flush():79] Configure stats pid to 335755 +2024-10-30 01:31:41,691 INFO MainThread:335755 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-10-30 01:31:41,692 INFO MainThread:335755 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-10-30 01:31:41,692 INFO MainThread:335755 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-10-30 01:31:41,692 INFO MainThread:335755 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-10-30 01:31:41,692 INFO MainThread:335755 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'} +2024-10-30 01:31:41,692 INFO MainThread:335755 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-10-30 01:31:41,692 INFO MainThread:335755 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_013141-bkcoggdw/logs/debug.log +2024-10-30 01:31:41,692 INFO MainThread:335755 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_013141-bkcoggdw/logs/debug-internal.log +2024-10-30 01:31:41,692 INFO MainThread:335755 [wandb_init.py:init():621] calling init triggers +2024-10-30 01:31:41,692 INFO MainThread:335755 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-10-30 01:31:41,692 INFO MainThread:335755 [wandb_init.py:init():671] starting backend +2024-10-30 01:31:41,692 INFO MainThread:335755 [wandb_init.py:init():675] sending inform_init request +2024-10-30 01:31:41,693 INFO MainThread:335755 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-10-30 01:31:41,693 INFO MainThread:335755 [wandb_init.py:init():688] backend started and connected +2024-10-30 01:31:41,695 INFO MainThread:335755 [wandb_init.py:init():783] updated telemetry +2024-10-30 01:31:41,721 INFO MainThread:335755 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-10-30 01:31:44,311 INFO MainThread:335755 [wandb_init.py:init():867] starting run threads in backend +2024-10-30 01:31:44,431 INFO MainThread:335755 [wandb_run.py:_console_start():2463] atexit reg +2024-10-30 01:31:44,431 INFO MainThread:335755 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-10-30 01:31:44,431 INFO MainThread:335755 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-10-30 01:31:44,431 INFO MainThread:335755 [wandb_run.py:_redirect():2401] Redirects installed. +2024-10-30 01:31:44,432 INFO MainThread:335755 [wandb_init.py:init():911] run started, returning control to user process +2024-10-30 01:31:44,433 INFO MainThread:335755 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_full', 'train_set': '10M', 'batch_size': 3, 'epoch': 7, 'seed': 0} +2024-10-30 01:33:16,641 WARNING MsgRouterThr:335755 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20241031_001055-sr4xke8e/files/output.log b/wandb/run-20241031_001055-sr4xke8e/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..a002fa67a96e7e23d95c728520d431d6b3cf8eea --- /dev/null +++ b/wandb/run-20241031_001055-sr4xke8e/files/output.log @@ -0,0 +1,13 @@ +Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:05<00:00, 2.82s/it] +tokenized_valid: Dataset({ + features: ['input_ids', 'attention_mask'], + num_rows: 600 +}) +/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +[2024-10-31 00:11:03,358] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-10-31 00:11:12,356] [INFO] [comm.py:652:init_distributed] cdb=None +Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination +Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root... +Loading extension module cpu_adam... +Time to load cpu_adam op: 5.375590562820435 seconds diff --git a/wandb/run-20241031_001055-sr4xke8e/files/requirements.txt b/wandb/run-20241031_001055-sr4xke8e/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..95a931302e269cc9e4fa5b719b6511f176ee2416 --- /dev/null +++ b/wandb/run-20241031_001055-sr4xke8e/files/requirements.txt @@ -0,0 +1,147 @@ +funcsigs==1.0.2 +sentry-sdk==2.17.0 +multiprocess==0.70.16 +numpy==1.26.2 +pluralizer==1.2.0 +debugpy==1.6.7 +nvidia-cudnn-cu11==8.5.0.96 +deepspeed==0.15.2 +data==0.4 +pandas==2.1.3 +tomli==2.0.1 +charset-normalizer==3.3.2 +attrs==24.2.0 +aiosignal==1.3.1 +fsspec==2023.10.0 +nvidia-cusparse-cu11==11.7.4.91 +zipp==3.12.0 +mypy-extensions==1.0.0 +datasets==3.0.1 +joblib==1.3.2 +hjson==3.1.0 +traitlets==5.7.1 +stack-data==0.6.0 +transformers==4.45.1 +sympy==1.11.1 +Pygments==2.15.0 +docker-pycreds==0.4.0 +dill==0.3.8 +wheel==0.44.0 +prompt-toolkit==3.0.30 +parso==0.8.3 +ipykernel==6.23.1 +pyarrow==17.0.0 +certifi==2023.11.17 +nvidia-cufft-cu11==10.9.0.58 +six==1.16.0 +pydantic==2.9.2 +click==8.1.7 +nest-asyncio==1.5.6 +gmpy2==2.1.0 +matplotlib==3.8.2 +scipy==1.11.4 +typing_extensions==4.12.2 +statsmodels==0.14.0 +huggingface-hub==0.25.0 +frozenlist==1.4.1 +gpustat==1.1.1 +nvidia-nvtx-cu11==11.7.91 +safetensors==0.4.5 +stanza==1.9.2 +decorator==5.1.1 +seaborn==0.13.0 +sentencepiece==0.2.0 +PyYAML==6.0.1 +black==24.8.0 +protobuf==4.25.1 +pickleshare==0.7.5 +peft==0.13.0 +triton==2.0.0 +nvidia-cuda-runtime-cu11==11.7.99 +Jinja2==3.1.2 +nvidia-cusolver-cu11==11.4.0.1 +executing==1.2.0 +jupyter_client==8.1.0 +pluggy==1.3.0 +cmake==3.30.3 +pytz==2023.3.post1 +aiohappyeyeballs==2.4.2 +kiwisolver==1.4.5 +py-cpuinfo==9.0.0 +Pillow==10.1.0 +ptyprocess==0.7.0 +importlib_resources==6.4.5 +GitPython==3.1.43 +importlib-metadata==6.0.0 +iniconfig==2.0.0 +scikit-learn==1.3.2 +exceptiongroup==1.1.0 +networkx==2.8.6 +accelerate==1.0.0 +nltk==3.8.1 +shutilwhich==1.1.0 +fonttools==4.45.1 +future==0.18.3 +aiohttp==3.10.6 +wcwidth==0.2.5 +idna==3.6 +filelock==3.12.2 +pathspec==0.12.1 +jupyter_core==5.1.0 +lit==18.1.8 +nvidia-curand-cu11==10.2.10.91 +nvidia-cublas-cu11==11.10.3.66 +nvidia-ml-py==12.560.30 +msgpack==1.1.0 +python-dateutil==2.8.2 +blessed==1.20.0 +packaging==23.0 +gitdb==4.0.11 +yarl==1.13.0 +emoji==2.8.0 +tzdata==2023.3 +cycler==0.12.1 +tornado==6.2 +backcall==0.2.0 +plotnine==0.12.4 +ninja==1.11.1.1 +latex==0.7.0 +wandb==0.18.5 +setproctitle==1.3.3 +threadpoolctl==3.2.0 +requests==2.32.3 +pyparsing==3.1.1 +smmap==5.0.1 +pyzmq==23.0.0 +async-timeout==4.0.3 +annotated-types==0.7.0 +matplotlib-inline==0.1.6 +latexcodec==1.0.0 +ipython==8.0.0 +patsy==0.5.3 +contourpy==1.2.0 +multidict==6.1.0 +mizani==0.9.3 +urllib3==2.1.0 +tokenizers==0.20.0 +MarkupSafe==2.1.2 +pip==24.2 +pexpect==4.8.0 +tqdm==4.66.5 +jedi==0.18.2 +pydantic_core==2.23.4 +tempdir==0.7.1 +mpmath==1.2.1 +setuptools==72.1.0 +pytest==7.4.3 +pure-eval==0.2.2 +psutil==5.9.1 +comm==0.1.2 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +regex==2023.10.3 +platformdirs==2.5.2 +asttokens==2.2.1 +torch==2.0.0 +nvidia-nccl-cu11==2.14.3 +xxhash==3.5.0 diff --git a/wandb/run-20241031_001055-sr4xke8e/logs/debug.log b/wandb/run-20241031_001055-sr4xke8e/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..9a32af53055f1212211925b9415162d3dd7ac55a --- /dev/null +++ b/wandb/run-20241031_001055-sr4xke8e/logs/debug.log @@ -0,0 +1,26 @@ +2024-10-31 00:10:55,611 INFO MainThread:479384 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-10-31 00:10:55,612 INFO MainThread:479384 [wandb_setup.py:_flush():79] Configure stats pid to 479384 +2024-10-31 00:10:55,612 INFO MainThread:479384 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-10-31 00:10:55,612 INFO MainThread:479384 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-10-31 00:10:55,612 INFO MainThread:479384 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-10-31 00:10:55,612 INFO MainThread:479384 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-10-31 00:10:55,612 INFO MainThread:479384 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'} +2024-10-31 00:10:55,612 INFO MainThread:479384 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-10-31 00:10:55,612 INFO MainThread:479384 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_001055-sr4xke8e/logs/debug.log +2024-10-31 00:10:55,612 INFO MainThread:479384 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_001055-sr4xke8e/logs/debug-internal.log +2024-10-31 00:10:55,612 INFO MainThread:479384 [wandb_init.py:init():621] calling init triggers +2024-10-31 00:10:55,612 INFO MainThread:479384 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-10-31 00:10:55,612 INFO MainThread:479384 [wandb_init.py:init():671] starting backend +2024-10-31 00:10:55,612 INFO MainThread:479384 [wandb_init.py:init():675] sending inform_init request +2024-10-31 00:10:55,613 INFO MainThread:479384 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-10-31 00:10:55,613 INFO MainThread:479384 [wandb_init.py:init():688] backend started and connected +2024-10-31 00:10:55,616 INFO MainThread:479384 [wandb_init.py:init():783] updated telemetry +2024-10-31 00:10:55,642 INFO MainThread:479384 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-10-31 00:10:55,916 INFO MainThread:479384 [wandb_init.py:init():867] starting run threads in backend +2024-10-31 00:10:56,007 INFO MainThread:479384 [wandb_run.py:_console_start():2463] atexit reg +2024-10-31 00:10:56,007 INFO MainThread:479384 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-10-31 00:10:56,007 INFO MainThread:479384 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-10-31 00:10:56,008 INFO MainThread:479384 [wandb_run.py:_redirect():2401] Redirects installed. +2024-10-31 00:10:56,010 INFO MainThread:479384 [wandb_init.py:init():911] run started, returning control to user process +2024-10-31 00:10:56,010 INFO MainThread:479384 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_full', 'train_set': '10M', 'batch_size': 3, 'epoch': 6, 'seed': 0, 'lr': 1e-05} diff --git a/wandb/run-20241031_114700-3cqkhntc/files/output.log b/wandb/run-20241031_114700-3cqkhntc/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..6e0e542aa94fa4a60b365be46b9dc39e7196be8a --- /dev/null +++ b/wandb/run-20241031_114700-3cqkhntc/files/output.log @@ -0,0 +1,220 @@ +model-00001-of-00002.safetensors: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████| 4.97G/4.97G [01:58<00:00, 42.0MB/s] +model-00002-of-00002.safetensors: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1.46G/1.46G [00:34<00:00, 42.6MB/s] +Downloading shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [02:32<00:00, 76.42s/it]▊| 1.46G/1.46G [00:34<00:00, 42.8MB/s] +Loading checkpoint shards: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:04<00:00, 2.31s/it] +generation_config.json: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 185/185 [00:00<00:00, 96.9kB/s] +tokenized_valid: Dataset({ + features: ['input_ids', 'attention_mask'], + num_rows: 600 +}) +/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +[2024-10-31 11:49:40,282] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-10-31 11:49:48,785] [INFO] [comm.py:652:init_distributed] cdb=None +[2024-10-31 11:49:48,785] [INFO] [comm.py:683:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl +Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher. +Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination +Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root... +Loading extension module cpu_adam... +Time to load cpu_adam op: 4.844248533248901 seconds +wandb: WARNING The `run_name` is currently set to the same value as `TrainingArguments.output_dir`. If this was not intended, please specify a different run name by setting the `TrainingArguments.run_name` parameter. + +{'loss': 2.7565, 'grad_norm': 0.0, 'learning_rate': 0.0001, 'epoch': 0.0} +{'loss': 2.8049, 'grad_norm': 0.0, 'learning_rate': 0.0001, 'epoch': 0.0} +{'loss': 2.7883, 'grad_norm': 0.0, 'learning_rate': 0.0001, 'epoch': 0.01} +{'loss': 2.7843, 'grad_norm': 0.0, 'learning_rate': 0.0001, 'epoch': 0.01} +{'loss': 2.7828, 'grad_norm': 0.0, 'learning_rate': 0.0001, 'epoch': 0.01} +{'loss': 2.7493, 'grad_norm': 0.0, 'learning_rate': 0.0001, 'epoch': 0.01} +{'loss': 2.7948, 'grad_norm': 0.0, 'learning_rate': 0.0001, 'epoch': 0.01} +{'loss': 2.7614, 'grad_norm': 0.0, 'learning_rate': 0.0001, 'epoch': 0.02} +{'loss': 2.7626, 'grad_norm': 0.0, 'learning_rate': 0.0001, 'epoch': 0.02} +{'loss': 2.7527, 'grad_norm': 0.0, 'learning_rate': 0.0001, 'epoch': 0.02} + +{'eval_loss': 2.809370756149292, 'eval_runtime': 12.6957, 'eval_samples_per_second': 47.26, 'eval_steps_per_second': 1.024, 'epoch': 0.02} +{'loss': 2.7977, 'grad_norm': 0.0, 'learning_rate': 0.0001, 'epoch': 0.02} +{'loss': 2.7658, 'grad_norm': 0.0, 'learning_rate': 0.0001, 'epoch': 0.02} +{'loss': 2.7787, 'grad_norm': 0.0, 'learning_rate': 0.0001, 'epoch': 0.03} +{'loss': 2.7744, 'grad_norm': 0.0, 'learning_rate': 0.0001, 'epoch': 0.03} +{'loss': 2.7916, 'grad_norm': 6.1797637939453125, 'learning_rate': 9.996577686516086e-05, 'epoch': 0.03} +{'loss': 4.6579, 'grad_norm': 6.1797637939453125, 'learning_rate': 9.996577686516086e-05, 'epoch': 0.03} +{'loss': 4.6189, 'grad_norm': 6.1797637939453125, 'learning_rate': 9.996577686516086e-05, 'epoch': 0.03} +{'loss': 4.5916, 'grad_norm': 6.1797637939453125, 'learning_rate': 9.996577686516086e-05, 'epoch': 0.04} +{'loss': 4.6312, 'grad_norm': 6.1797637939453125, 'learning_rate': 9.996577686516086e-05, 'epoch': 0.04} +{'loss': 4.6558, 'grad_norm': 6.1797637939453125, 'learning_rate': 9.996577686516086e-05, 'epoch': 0.04} +{'eval_loss': 4.709869861602783, 'eval_runtime': 11.5289, 'eval_samples_per_second': 52.043, 'eval_steps_per_second': 1.128, 'epoch': 0.04} +{'loss': 4.6179, 'grad_norm': 85.80337524414062, 'learning_rate': 9.99315537303217e-05, 'epoch': 0.04} +{'loss': 9.1321, 'grad_norm': 85.80337524414062, 'learning_rate': 9.99315537303217e-05, 'epoch': 0.05} +{'loss': 9.0391, 'grad_norm': 85.80337524414062, 'learning_rate': 9.99315537303217e-05, 'epoch': 0.05} +{'loss': 9.0784, 'grad_norm': 85.80337524414062, 'learning_rate': 9.99315537303217e-05, 'epoch': 0.05} +{'loss': 9.0339, 'grad_norm': 340.5785827636719, 'learning_rate': 9.989733059548256e-05, 'epoch': 0.05} +{'loss': 4.5744, 'grad_norm': 94.66028594970703, 'learning_rate': 9.98631074606434e-05, 'epoch': 0.05} +{'loss': 3.428, 'grad_norm': 36.625179290771484, 'learning_rate': 9.982888432580426e-05, 'epoch': 0.06} +{'loss': 2.8456, 'grad_norm': 16.80606460571289, 'learning_rate': 9.97946611909651e-05, 'epoch': 0.06} +{'loss': 2.6428, 'grad_norm': 9.26777458190918, 'learning_rate': 9.976043805612595e-05, 'epoch': 0.06} +{'loss': 2.6328, 'grad_norm': 9.187769889831543, 'learning_rate': 9.97262149212868e-05, 'epoch': 0.06} +{'eval_loss': 2.725268602371216, 'eval_runtime': 11.5568, 'eval_samples_per_second': 51.918, 'eval_steps_per_second': 1.125, 'epoch': 0.06} +{'loss': 2.5229, 'grad_norm': 6.6883864402771, 'learning_rate': 9.969199178644765e-05, 'epoch': 0.06} +{'loss': 2.5127, 'grad_norm': 2.9667153358459473, 'learning_rate': 9.965776865160849e-05, 'epoch': 0.07} +{'loss': 2.4111, 'grad_norm': 6.198004245758057, 'learning_rate': 9.962354551676935e-05, 'epoch': 0.07} +{'loss': 2.4049, 'grad_norm': 2.4675652980804443, 'learning_rate': 9.958932238193019e-05, 'epoch': 0.07} +{'loss': 2.4398, 'grad_norm': 3.319000720977783, 'learning_rate': 9.955509924709105e-05, 'epoch': 0.07} +{'loss': 2.3516, 'grad_norm': 1.6354080438613892, 'learning_rate': 9.952087611225189e-05, 'epoch': 0.07} +{'loss': 2.3761, 'grad_norm': 3.3287758827209473, 'learning_rate': 9.948665297741273e-05, 'epoch': 0.08} +{'loss': 2.3453, 'grad_norm': 2.034022331237793, 'learning_rate': 9.945242984257359e-05, 'epoch': 0.08} +{'loss': 2.3651, 'grad_norm': 2.3366262912750244, 'learning_rate': 9.941820670773443e-05, 'epoch': 0.08} +{'loss': 2.3091, 'grad_norm': 2.418919801712036, 'learning_rate': 9.938398357289528e-05, 'epoch': 0.08} +{'eval_loss': 2.5406363010406494, 'eval_runtime': 11.5689, 'eval_samples_per_second': 51.863, 'eval_steps_per_second': 1.124, 'epoch': 0.08} +{'loss': 2.2928, 'grad_norm': 1.524388313293457, 'learning_rate': 9.934976043805613e-05, 'epoch': 0.08} +{'loss': 2.2994, 'grad_norm': 3.2633249759674072, 'learning_rate': 9.931553730321698e-05, 'epoch': 0.09} +{'loss': 2.2726, 'grad_norm': 1.2831145524978638, 'learning_rate': 9.928131416837782e-05, 'epoch': 0.09} +{'loss': 2.2493, 'grad_norm': 2.7639758586883545, 'learning_rate': 9.924709103353868e-05, 'epoch': 0.09} +{'loss': 2.2367, 'grad_norm': 1.9813640117645264, 'learning_rate': 9.921286789869952e-05, 'epoch': 0.09} +{'loss': 2.2335, 'grad_norm': 1.9555699825286865, 'learning_rate': 9.917864476386038e-05, 'epoch': 0.09} +{'loss': 2.2102, 'grad_norm': 2.355908155441284, 'learning_rate': 9.914442162902122e-05, 'epoch': 0.1} +{'loss': 2.2115, 'grad_norm': 2.112484931945801, 'learning_rate': 9.911019849418207e-05, 'epoch': 0.1} +{'loss': 2.1491, 'grad_norm': 1.4058802127838135, 'learning_rate': 9.907597535934292e-05, 'epoch': 0.1} +{'loss': 2.1466, 'grad_norm': 2.0998988151550293, 'learning_rate': 9.904175222450377e-05, 'epoch': 0.1} +{'eval_loss': 2.4130947589874268, 'eval_runtime': 11.5707, 'eval_samples_per_second': 51.855, 'eval_steps_per_second': 1.124, 'epoch': 0.1} +{'loss': 2.1452, 'grad_norm': 1.3756946325302124, 'learning_rate': 9.900752908966461e-05, 'epoch': 0.1} +{'loss': 2.1067, 'grad_norm': 1.696256399154663, 'learning_rate': 9.897330595482547e-05, 'epoch': 0.11} +{'loss': 2.1575, 'grad_norm': 1.313455581665039, 'learning_rate': 9.893908281998631e-05, 'epoch': 0.11} +{'loss': 2.1548, 'grad_norm': 1.4613757133483887, 'learning_rate': 9.890485968514717e-05, 'epoch': 0.11} +{'loss': 2.0482, 'grad_norm': 1.3651756048202515, 'learning_rate': 9.887063655030801e-05, 'epoch': 0.11} +{'loss': 2.0557, 'grad_norm': 0.923547089099884, 'learning_rate': 9.883641341546886e-05, 'epoch': 0.11} +{'loss': 2.0637, 'grad_norm': 1.3289296627044678, 'learning_rate': 9.880219028062972e-05, 'epoch': 0.12} +{'loss': 2.064, 'grad_norm': 1.094741940498352, 'learning_rate': 9.876796714579056e-05, 'epoch': 0.12} +{'loss': 2.0745, 'grad_norm': 1.7548996210098267, 'learning_rate': 9.873374401095142e-05, 'epoch': 0.12} +{'loss': 2.0973, 'grad_norm': 1.5266579389572144, 'learning_rate': 9.869952087611226e-05, 'epoch': 0.12} +{'eval_loss': 2.3453779220581055, 'eval_runtime': 11.5705, 'eval_samples_per_second': 51.856, 'eval_steps_per_second': 1.124, 'epoch': 0.12} +{'loss': 2.0411, 'grad_norm': 0.9586588144302368, 'learning_rate': 9.866529774127312e-05, 'epoch': 0.13} +{'loss': 1.9945, 'grad_norm': 1.53722083568573, 'learning_rate': 9.863107460643394e-05, 'epoch': 0.13} +{'loss': 2.0012, 'grad_norm': 1.1385856866836548, 'learning_rate': 9.85968514715948e-05, 'epoch': 0.13} +{'loss': 1.9785, 'grad_norm': 1.4056907892227173, 'learning_rate': 9.856262833675564e-05, 'epoch': 0.13} +{'loss': 2.0375, 'grad_norm': 1.1709682941436768, 'learning_rate': 9.85284052019165e-05, 'epoch': 0.13} +{'loss': 1.9961, 'grad_norm': 1.3510781526565552, 'learning_rate': 9.849418206707734e-05, 'epoch': 0.14} +{'loss': 1.9745, 'grad_norm': 1.6425883769989014, 'learning_rate': 9.84599589322382e-05, 'epoch': 0.14} +{'loss': 1.9905, 'grad_norm': 1.2176721096038818, 'learning_rate': 9.842573579739904e-05, 'epoch': 0.14} +{'loss': 1.9421, 'grad_norm': 1.3667473793029785, 'learning_rate': 9.839151266255989e-05, 'epoch': 0.14} +{'loss': 1.9452, 'grad_norm': 1.8916258811950684, 'learning_rate': 9.835728952772075e-05, 'epoch': 0.14} +{'eval_loss': 2.301532506942749, 'eval_runtime': 11.5625, 'eval_samples_per_second': 51.892, 'eval_steps_per_second': 1.124, 'epoch': 0.14} +{'loss': 1.9262, 'grad_norm': 1.7627770900726318, 'learning_rate': 9.832306639288159e-05, 'epoch': 0.15} +{'loss': 1.9736, 'grad_norm': 1.951368808746338, 'learning_rate': 9.828884325804245e-05, 'epoch': 0.15} +{'loss': 1.8966, 'grad_norm': 2.173610210418701, 'learning_rate': 9.825462012320329e-05, 'epoch': 0.15} +{'loss': 1.9133, 'grad_norm': 1.0655031204223633, 'learning_rate': 9.822039698836414e-05, 'epoch': 0.15} +{'loss': 1.9703, 'grad_norm': 1.9031049013137817, 'learning_rate': 9.818617385352499e-05, 'epoch': 0.15} +{'loss': 1.8632, 'grad_norm': 1.8678045272827148, 'learning_rate': 9.815195071868584e-05, 'epoch': 0.16} +{'loss': 1.9308, 'grad_norm': 1.3070083856582642, 'learning_rate': 9.811772758384668e-05, 'epoch': 0.16} +{'loss': 1.8469, 'grad_norm': 2.6512370109558105, 'learning_rate': 9.808350444900754e-05, 'epoch': 0.16} +{'loss': 1.9147, 'grad_norm': 2.861356735229492, 'learning_rate': 9.804928131416838e-05, 'epoch': 0.16} +{'loss': 1.8943, 'grad_norm': 1.9181678295135498, 'learning_rate': 9.801505817932924e-05, 'epoch': 0.16} +{'eval_loss': 2.289658784866333, 'eval_runtime': 11.5613, 'eval_samples_per_second': 51.897, 'eval_steps_per_second': 1.124, 'epoch': 0.16} +{'loss': 1.8777, 'grad_norm': 2.2638590335845947, 'learning_rate': 9.798083504449008e-05, 'epoch': 0.17} +{'loss': 1.8578, 'grad_norm': 2.1869819164276123, 'learning_rate': 9.794661190965093e-05, 'epoch': 0.17} +{'loss': 1.8953, 'grad_norm': 1.6545476913452148, 'learning_rate': 9.791238877481178e-05, 'epoch': 0.17} +{'loss': 1.8593, 'grad_norm': 1.9989372491836548, 'learning_rate': 9.787816563997263e-05, 'epoch': 0.17} +{'loss': 1.904, 'grad_norm': 1.3267912864685059, 'learning_rate': 9.784394250513347e-05, 'epoch': 0.17} +{'loss': 1.8325, 'grad_norm': 1.8499799966812134, 'learning_rate': 9.780971937029433e-05, 'epoch': 0.18} +{'loss': 1.7943, 'grad_norm': 1.453418254852295, 'learning_rate': 9.777549623545517e-05, 'epoch': 0.18} +{'loss': 1.8924, 'grad_norm': 1.9336315393447876, 'learning_rate': 9.774127310061603e-05, 'epoch': 0.18} +{'loss': 1.8241, 'grad_norm': 1.2739551067352295, 'learning_rate': 9.770704996577687e-05, 'epoch': 0.18} +{'loss': 1.8327, 'grad_norm': 1.8302478790283203, 'learning_rate': 9.767282683093771e-05, 'epoch': 0.18} +{'eval_loss': 2.255014181137085, 'eval_runtime': 11.5686, 'eval_samples_per_second': 51.865, 'eval_steps_per_second': 1.124, 'epoch': 0.18} +{'loss': 1.8746, 'grad_norm': 1.3933634757995605, 'learning_rate': 9.763860369609857e-05, 'epoch': 0.19} +{'loss': 1.8156, 'grad_norm': 1.53447425365448, 'learning_rate': 9.760438056125941e-05, 'epoch': 0.19} +{'loss': 1.7668, 'grad_norm': 1.744211196899414, 'learning_rate': 9.757015742642026e-05, 'epoch': 0.19} +{'loss': 1.8081, 'grad_norm': 1.784866213798523, 'learning_rate': 9.75359342915811e-05, 'epoch': 0.19} +{'loss': 1.7898, 'grad_norm': 1.3280785083770752, 'learning_rate': 9.750171115674196e-05, 'epoch': 0.2} +{'loss': 1.8135, 'grad_norm': 1.3295087814331055, 'learning_rate': 9.74674880219028e-05, 'epoch': 0.2} +{'loss': 1.784, 'grad_norm': 1.3081759214401245, 'learning_rate': 9.743326488706366e-05, 'epoch': 0.2} +{'loss': 1.7673, 'grad_norm': 1.2620002031326294, 'learning_rate': 9.73990417522245e-05, 'epoch': 0.2} +{'loss': 1.7492, 'grad_norm': 1.2635165452957153, 'learning_rate': 9.736481861738536e-05, 'epoch': 0.2} +{'loss': 1.8075, 'grad_norm': 1.3014883995056152, 'learning_rate': 9.73305954825462e-05, 'epoch': 0.21} +{'eval_loss': 2.2504258155822754, 'eval_runtime': 11.5687, 'eval_samples_per_second': 51.864, 'eval_steps_per_second': 1.124, 'epoch': 0.21} +{'loss': 1.7359, 'grad_norm': 1.090064287185669, 'learning_rate': 9.729637234770705e-05, 'epoch': 0.21} +{'loss': 1.7055, 'grad_norm': 1.180134654045105, 'learning_rate': 9.726214921286791e-05, 'epoch': 0.21} +{'loss': 1.7171, 'grad_norm': 1.392907738685608, 'learning_rate': 9.722792607802875e-05, 'epoch': 0.21} +{'loss': 1.7232, 'grad_norm': 1.2521458864212036, 'learning_rate': 9.719370294318961e-05, 'epoch': 0.21} +{'loss': 1.7408, 'grad_norm': 1.3325889110565186, 'learning_rate': 9.715947980835045e-05, 'epoch': 0.22} +{'loss': 1.8078, 'grad_norm': 1.593698501586914, 'learning_rate': 9.71252566735113e-05, 'epoch': 0.22} +{'loss': 1.7849, 'grad_norm': 1.251745343208313, 'learning_rate': 9.709103353867215e-05, 'epoch': 0.22} +{'loss': 1.7266, 'grad_norm': 1.555539608001709, 'learning_rate': 9.7056810403833e-05, 'epoch': 0.22} +{'loss': 1.7433, 'grad_norm': 1.5724241733551025, 'learning_rate': 9.702258726899384e-05, 'epoch': 0.22} +{'loss': 1.7508, 'grad_norm': 1.3853074312210083, 'learning_rate': 9.69883641341547e-05, 'epoch': 0.23} +{'eval_loss': 2.18611216545105, 'eval_runtime': 11.5643, 'eval_samples_per_second': 51.884, 'eval_steps_per_second': 1.124, 'epoch': 0.23} +{'loss': 1.7278, 'grad_norm': 1.2098917961120605, 'learning_rate': 9.695414099931554e-05, 'epoch': 0.23} +{'loss': 1.7654, 'grad_norm': 1.171714425086975, 'learning_rate': 9.69199178644764e-05, 'epoch': 0.23} +{'loss': 1.741, 'grad_norm': 1.2655503749847412, 'learning_rate': 9.688569472963724e-05, 'epoch': 0.23} +{'loss': 1.7532, 'grad_norm': 1.1870505809783936, 'learning_rate': 9.68514715947981e-05, 'epoch': 0.23} +{'loss': 1.7446, 'grad_norm': 1.2463093996047974, 'learning_rate': 9.681724845995894e-05, 'epoch': 0.24} +{'loss': 1.7164, 'grad_norm': 1.0629775524139404, 'learning_rate': 9.678302532511978e-05, 'epoch': 0.24} +{'loss': 1.7579, 'grad_norm': 1.4846782684326172, 'learning_rate': 9.674880219028064e-05, 'epoch': 0.24} +{'loss': 1.7606, 'grad_norm': 1.371062159538269, 'learning_rate': 9.671457905544148e-05, 'epoch': 0.24} +{'loss': 1.7149, 'grad_norm': 1.347663164138794, 'learning_rate': 9.668035592060233e-05, 'epoch': 0.24} +{'loss': 1.738, 'grad_norm': 1.2462127208709717, 'learning_rate': 9.664613278576318e-05, 'epoch': 0.25} +{'eval_loss': 2.1583096981048584, 'eval_runtime': 11.5652, 'eval_samples_per_second': 51.88, 'eval_steps_per_second': 1.124, 'epoch': 0.25} +{'loss': 1.7279, 'grad_norm': 1.3426861763000488, 'learning_rate': 9.661190965092403e-05, 'epoch': 0.25} +{'loss': 1.6961, 'grad_norm': 1.3338221311569214, 'learning_rate': 9.657768651608487e-05, 'epoch': 0.25} +{'loss': 1.6725, 'grad_norm': 1.252794623374939, 'learning_rate': 9.654346338124573e-05, 'epoch': 0.25} +{'loss': 1.7193, 'grad_norm': 1.3079990148544312, 'learning_rate': 9.650924024640657e-05, 'epoch': 0.25} +{'loss': 1.7285, 'grad_norm': 1.2246983051300049, 'learning_rate': 9.647501711156743e-05, 'epoch': 0.26} +{'loss': 1.7116, 'grad_norm': 1.2157689332962036, 'learning_rate': 9.644079397672827e-05, 'epoch': 0.26} +{'loss': 1.7175, 'grad_norm': 1.3069276809692383, 'learning_rate': 9.640657084188912e-05, 'epoch': 0.26} +{'loss': 1.7227, 'grad_norm': 1.3949460983276367, 'learning_rate': 9.637234770704997e-05, 'epoch': 0.26} +{'loss': 1.7351, 'grad_norm': 1.346846580505371, 'learning_rate': 9.633812457221082e-05, 'epoch': 0.26} +{'loss': 1.7212, 'grad_norm': 1.3208940029144287, 'learning_rate': 9.630390143737166e-05, 'epoch': 0.27} +{'eval_loss': 2.1235294342041016, 'eval_runtime': 11.5634, 'eval_samples_per_second': 51.888, 'eval_steps_per_second': 1.124, 'epoch': 0.27} +{'loss': 1.6919, 'grad_norm': 1.1460075378417969, 'learning_rate': 9.626967830253252e-05, 'epoch': 0.27} +{'loss': 1.6896, 'grad_norm': 1.5786269903182983, 'learning_rate': 9.623545516769336e-05, 'epoch': 0.27} +{'loss': 1.7246, 'grad_norm': 1.4066743850708008, 'learning_rate': 9.620123203285422e-05, 'epoch': 0.27} +{'loss': 1.6981, 'grad_norm': 1.4564768075942993, 'learning_rate': 9.616700889801506e-05, 'epoch': 0.28} +{'loss': 1.6919, 'grad_norm': 1.3670514822006226, 'learning_rate': 9.613278576317591e-05, 'epoch': 0.28} +{'loss': 1.7132, 'grad_norm': 1.290029525756836, 'learning_rate': 9.609856262833677e-05, 'epoch': 0.28} +{'loss': 1.686, 'grad_norm': 1.171757459640503, 'learning_rate': 9.606433949349761e-05, 'epoch': 0.28} +{'loss': 1.667, 'grad_norm': 1.3537696599960327, 'learning_rate': 9.603011635865847e-05, 'epoch': 0.28} +{'loss': 1.704, 'grad_norm': 1.4693126678466797, 'learning_rate': 9.599589322381931e-05, 'epoch': 0.29} +{'loss': 1.6503, 'grad_norm': 1.2005213499069214, 'learning_rate': 9.596167008898017e-05, 'epoch': 0.29} +{'eval_loss': 2.1207213401794434, 'eval_runtime': 11.5721, 'eval_samples_per_second': 51.849, 'eval_steps_per_second': 1.123, 'epoch': 0.29} +{'loss': 1.7115, 'grad_norm': 1.1947609186172485, 'learning_rate': 9.592744695414101e-05, 'epoch': 0.29} +{'loss': 1.6809, 'grad_norm': 1.4202208518981934, 'learning_rate': 9.589322381930185e-05, 'epoch': 0.29} +{'loss': 1.682, 'grad_norm': 1.5427730083465576, 'learning_rate': 9.585900068446269e-05, 'epoch': 0.29} +{'loss': 1.6823, 'grad_norm': 1.2999160289764404, 'learning_rate': 9.582477754962355e-05, 'epoch': 0.3} +{'loss': 1.6988, 'grad_norm': 1.2814061641693115, 'learning_rate': 9.579055441478439e-05, 'epoch': 0.3} +{'loss': 1.6891, 'grad_norm': 1.311761736869812, 'learning_rate': 9.575633127994524e-05, 'epoch': 0.3} +{'loss': 1.6682, 'grad_norm': 1.2029725313186646, 'learning_rate': 9.572210814510609e-05, 'epoch': 0.3} +{'loss': 1.67, 'grad_norm': 1.0917092561721802, 'learning_rate': 9.568788501026694e-05, 'epoch': 0.3} +{'loss': 1.6268, 'grad_norm': 1.335014820098877, 'learning_rate': 9.56536618754278e-05, 'epoch': 0.31} +{'loss': 1.6501, 'grad_norm': 1.3213744163513184, 'learning_rate': 9.561943874058864e-05, 'epoch': 0.31} +{'eval_loss': 2.103264570236206, 'eval_runtime': 11.5773, 'eval_samples_per_second': 51.826, 'eval_steps_per_second': 1.123, 'epoch': 0.31} +{'loss': 1.6835, 'grad_norm': 1.2449109554290771, 'learning_rate': 9.55852156057495e-05, 'epoch': 0.31} +{'loss': 1.6984, 'grad_norm': 1.4669255018234253, 'learning_rate': 9.555099247091034e-05, 'epoch': 0.31} +{'loss': 1.6771, 'grad_norm': 1.2852448225021362, 'learning_rate': 9.551676933607119e-05, 'epoch': 0.31} +{'loss': 1.6275, 'grad_norm': 1.3089197874069214, 'learning_rate': 9.548254620123203e-05, 'epoch': 0.32} +{'loss': 1.6546, 'grad_norm': 1.453117847442627, 'learning_rate': 9.544832306639289e-05, 'epoch': 0.32} +{'loss': 1.6131, 'grad_norm': 1.1836069822311401, 'learning_rate': 9.541409993155373e-05, 'epoch': 0.32} +{'loss': 1.6603, 'grad_norm': 1.197521686553955, 'learning_rate': 9.537987679671459e-05, 'epoch': 0.32} +{'loss': 1.6618, 'grad_norm': 1.3445478677749634, 'learning_rate': 9.534565366187543e-05, 'epoch': 0.32} +{'loss': 1.6263, 'grad_norm': 1.3089582920074463, 'learning_rate': 9.531143052703629e-05, 'epoch': 0.33} + trainer.train() + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/trainer.py", line 2052, in train + return inner_training_loop( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/trainer.py", line 2388, in _inner_training_loop + tr_loss_step = self.training_step(model, inputs) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/trainer.py", line 3518, in training_step + self.accelerator.backward(loss, **kwargs) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/accelerate/accelerator.py", line 2238, in backward + self.deepspeed_engine_wrapped.backward(loss, **kwargs) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/accelerate/utils/deepspeed.py", line 186, in backward + self.engine.backward(loss, **kwargs) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/utils/nvtx.py", line 18, in wrapped_fn + ret_val = func(*args, **kwargs) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/runtime/engine.py", line 2020, in backward + self.optimizer.backward(loss, retain_graph=retain_graph) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/runtime/zero/stage_1_and_2.py", line 2063, in backward + self.loss_scaler.backward(loss.float(), retain_graph=retain_graph) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/runtime/fp16/loss_scaler.py", line 63, in backward + scaled_loss.backward(retain_graph=retain_graph) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/torch/_tensor.py", line 487, in backward + torch.autograd.backward( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/torch/autograd/__init__.py", line 200, in backward + Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass +KeyboardInterrupt diff --git a/wandb/run-20241031_114700-3cqkhntc/logs/debug-internal.log b/wandb/run-20241031_114700-3cqkhntc/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..3e34976a1e71e9defffb600de5510141c2b32d4a --- /dev/null +++ b/wandb/run-20241031_114700-3cqkhntc/logs/debug-internal.log @@ -0,0 +1,8 @@ +{"time":"2024-10-31T11:47:00.291622632-04:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-10-31T11:47:00.291649983-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_114700-3cqkhntc/logs/debug-core.log"} +{"time":"2024-10-31T11:47:00.400110384-04:00","level":"INFO","msg":"created new stream","id":"3cqkhntc"} +{"time":"2024-10-31T11:47:00.400155725-04:00","level":"INFO","msg":"stream: started","id":"3cqkhntc"} +{"time":"2024-10-31T11:47:00.400207845-04:00","level":"INFO","msg":"sender: started","stream_id":"3cqkhntc"} +{"time":"2024-10-31T11:47:00.400176745-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"3cqkhntc"}} +{"time":"2024-10-31T11:47:00.400210445-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"3cqkhntc"}} +{"time":"2024-10-31T11:47:00.639856673-04:00","level":"INFO","msg":"Starting system monitor"} diff --git a/wandb/run-20241031_114700-3cqkhntc/logs/debug.log b/wandb/run-20241031_114700-3cqkhntc/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..5ebdfafa85eb2d1687a6a002d01d5db2a431e61f --- /dev/null +++ b/wandb/run-20241031_114700-3cqkhntc/logs/debug.log @@ -0,0 +1,29 @@ +2024-10-31 11:47:00,285 INFO MainThread:554143 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-10-31 11:47:00,286 INFO MainThread:554143 [wandb_setup.py:_flush():79] Configure stats pid to 554143 +2024-10-31 11:47:00,286 INFO MainThread:554143 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-10-31 11:47:00,286 INFO MainThread:554143 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-10-31 11:47:00,286 INFO MainThread:554143 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-10-31 11:47:00,286 INFO MainThread:554143 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-10-31 11:47:00,286 INFO MainThread:554143 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'} +2024-10-31 11:47:00,286 INFO MainThread:554143 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-10-31 11:47:00,286 INFO MainThread:554143 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_114700-3cqkhntc/logs/debug.log +2024-10-31 11:47:00,286 INFO MainThread:554143 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_114700-3cqkhntc/logs/debug-internal.log +2024-10-31 11:47:00,286 INFO MainThread:554143 [wandb_init.py:init():621] calling init triggers +2024-10-31 11:47:00,286 INFO MainThread:554143 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-10-31 11:47:00,286 INFO MainThread:554143 [wandb_init.py:init():671] starting backend +2024-10-31 11:47:00,286 INFO MainThread:554143 [wandb_init.py:init():675] sending inform_init request +2024-10-31 11:47:00,288 INFO MainThread:554143 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-10-31 11:47:00,288 INFO MainThread:554143 [wandb_init.py:init():688] backend started and connected +2024-10-31 11:47:00,293 INFO MainThread:554143 [wandb_init.py:init():783] updated telemetry +2024-10-31 11:47:00,326 INFO MainThread:554143 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-10-31 11:47:00,636 INFO MainThread:554143 [wandb_init.py:init():867] starting run threads in backend +2024-10-31 11:47:00,724 INFO MainThread:554143 [wandb_run.py:_console_start():2463] atexit reg +2024-10-31 11:47:00,724 INFO MainThread:554143 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-10-31 11:47:00,724 INFO MainThread:554143 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-10-31 11:47:00,724 INFO MainThread:554143 [wandb_run.py:_redirect():2401] Redirects installed. +2024-10-31 11:47:00,725 INFO MainThread:554143 [wandb_init.py:init():911] run started, returning control to user process +2024-10-31 11:47:00,725 INFO MainThread:554143 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_full', 'train_set': '10M', 'batch_size': 3, 'epoch': 6, 'seed': 0, 'lr': 0.0001} +2024-10-31 11:50:15,611 INFO MainThread:554143 [wandb_run.py:_config_callback():1390] config_cb None None {'vocab_size': 128256, 'max_position_embeddings': 131072, 'hidden_size': 3072, 'intermediate_size': 8192, 'num_hidden_layers': 28, 'num_attention_heads': 24, 'num_key_value_heads': 8, 'hidden_act': 'silu', 'initializer_range': 0.02, 'rms_norm_eps': 1e-05, 'pretraining_tp': 1, 'use_cache': True, 'rope_theta': 500000.0, 'rope_scaling': {'factor': 32.0, 'high_freq_factor': 4.0, 'low_freq_factor': 1.0, 'original_max_position_embeddings': 8192, 'rope_type': 'llama3'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 128, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'bfloat16', 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': True, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['LlamaForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 128000, 'pad_token_id': None, 'eos_token_id': 128001, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'meta-llama/Llama-3.2-3B', 'transformers_version': '4.45.1', 'model_type': 'llama', 'output_dir': './checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': True, 'do_predict': False, 'eval_strategy': 'steps', 'prediction_loss_only': False, 'per_device_train_batch_size': 3, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 2, 'eval_accumulation_steps': None, 'eval_delay': 0, 'torch_empty_cache_steps': None, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 6, 'max_steps': -1, 'lr_scheduler_type': 'linear', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.0, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': './logs', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 1, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 150, 'save_total_limit': None, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 0, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': False, 'fp16': True, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 10, 'dataloader_num_workers': 0, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': './checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': 'deepspeed_config/train_dp_config.json', 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'evaluation_strategy': 'steps', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False, 'use_liger_kernel': False, 'eval_use_gather_object': False} +2024-10-31 11:50:15,613 INFO MainThread:554143 [wandb_config.py:__setitem__():154] config set model/num_parameters = 3212749824 - > +2024-10-31 11:50:15,613 INFO MainThread:554143 [wandb_run.py:_config_callback():1390] config_cb model/num_parameters 3212749824 None diff --git a/wandb/run-20241031_114700-q0d78n2b/files/output.log b/wandb/run-20241031_114700-q0d78n2b/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..33788c4e7ff11bb4e2789046d3b341e18b007346 --- /dev/null +++ b/wandb/run-20241031_114700-q0d78n2b/files/output.log @@ -0,0 +1,43 @@ +Downloading shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [02:32<00:00, 76.42s/it] +Loading checkpoint shards: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:04<00:00, 2.36s/it] +tokenized_valid: Dataset({ + features: ['input_ids', 'attention_mask'], + num_rows: 600 +}) +/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +[2024-10-31 11:49:40,269] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-10-31 11:49:48,813] [INFO] [comm.py:652:init_distributed] cdb=None +Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination +Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root... +Loading extension module cpu_adam... +Time to load cpu_adam op: 4.811358213424683 seconds +Traceback (most recent call last): + File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 220, in + trainer.train() + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/trainer.py", line 2052, in train + return inner_training_loop( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/trainer.py", line 2388, in _inner_training_loop + tr_loss_step = self.training_step(model, inputs) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/trainer.py", line 3518, in training_step + self.accelerator.backward(loss, **kwargs) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/accelerate/accelerator.py", line 2238, in backward + self.deepspeed_engine_wrapped.backward(loss, **kwargs) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/accelerate/utils/deepspeed.py", line 186, in backward + self.engine.backward(loss, **kwargs) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/utils/nvtx.py", line 18, in wrapped_fn + ret_val = func(*args, **kwargs) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/runtime/engine.py", line 2020, in backward + self.optimizer.backward(loss, retain_graph=retain_graph) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/runtime/zero/stage_1_and_2.py", line 2063, in backward + self.loss_scaler.backward(loss.float(), retain_graph=retain_graph) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/runtime/fp16/loss_scaler.py", line 63, in backward + scaled_loss.backward(retain_graph=retain_graph) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/torch/_tensor.py", line 487, in backward + torch.autograd.backward( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/torch/autograd/__init__.py", line 200, in backward + Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass +KeyboardInterrupt +Error in atexit._run_exitfuncs: +Traceback (most recent call last): + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/ops/transformer/inference/triton/matmul_ext.py", line 27, in is_nfs_path diff --git a/wandb/run-20241101_012438-1onnw7d5/files/config.yaml b/wandb/run-20241101_012438-1onnw7d5/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16d25ee0e4092474360045faaf0959cfc1b3e91d --- /dev/null +++ b/wandb/run-20241101_012438-1onnw7d5/files/config.yaml @@ -0,0 +1,49 @@ +_wandb: + value: + cli_version: 0.18.5 + m: [] + python_version: 3.9.19 + t: + "1": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "2": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "3": + - 13 + - 23 + - 55 + "4": 3.9.19 + "5": 0.18.5 + "6": 4.45.1 + "8": + - 5 + "12": 0.18.5 + "13": linux-x86_64 +batch_size: + value: 3 +epoch: + value: 6 +lr: + value: 5e-06 +perturbation: + value: shuffle_nodeterministic +seed: + value: 0 +train_set: + value: 10M diff --git a/wandb/run-20241101_012438-1onnw7d5/files/output.log b/wandb/run-20241101_012438-1onnw7d5/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..5824b3c1b0e7410256b3374a200db5bd3cc11c9d --- /dev/null +++ b/wandb/run-20241101_012438-1onnw7d5/files/output.log @@ -0,0 +1,12 @@ +Traceback (most recent call last): + File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 164, in + dataset = load_dataset('babylm_dataset_test.py', name=dataset_name, trust_remote_code=True) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/load.py", line 2074, in load_dataset + builder_instance = load_dataset_builder( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/load.py", line 1832, in load_dataset_builder + builder_instance: DatasetBuilder = builder_cls( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/builder.py", line 342, in __init__ + self.config, self.config_id = self._create_builder_config( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/builder.py", line 569, in _create_builder_config + raise ValueError( +ValueError: BuilderConfig 'babylm_shuffle_nodeterministic_10M_seed0' not found. Available: ['babylm_hop_control_10M_seed0', 'babylm_hop_tokens4_10M_seed0', 'babylm_hop_words4_10M_seed0', 'babylm_reverse_control_10M_seed0', 'babylm_reverse_partial_10M_seed0', 'babylm_reverse_full_10M_seed0', 'babylm_shuffle_control_10M_seed0', 'babylm_shuffle_nondeterministic_10M_seed0', 'babylm_shuffle_deterministic21_10M_seed0', 'babylm_shuffle_deterministic57_10M_seed0', 'babylm_shuffle_deterministic84_10M_seed0', 'babylm_shuffle_local3_10M_seed0', 'babylm_shuffle_local5_10M_seed0', 'babylm_shuffle_local10_10M_seed0', 'babylm_shuffle_even_odd_10M_seed0'] diff --git a/wandb/run-20241101_012438-1onnw7d5/files/wandb-metadata.json b/wandb/run-20241101_012438-1onnw7d5/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..a6d983f3e99ffc2ef807ffb839f0908a00e100ae --- /dev/null +++ b/wandb/run-20241101_012438-1onnw7d5/files/wandb-metadata.json @@ -0,0 +1,97 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-11-01T05:24:38.172065Z", + "args": [ + "--perturbation", + "shuffle_nodeterministic", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "6", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", + "codePath": "train/train_deep_wandb.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_deep_wandb.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA RTX A6000", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888559353856", + "used": "1753992159232" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + } + ], + "cudaVersion": "11.8" +} \ No newline at end of file diff --git a/wandb/run-20241101_012438-1onnw7d5/files/wandb-summary.json b/wandb/run-20241101_012438-1onnw7d5/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..6c37fe1cbbb8aed86fd461a79642cb991e4d35cf --- /dev/null +++ b/wandb/run-20241101_012438-1onnw7d5/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":0}} \ No newline at end of file diff --git a/wandb/run-20241101_012438-1onnw7d5/logs/debug-internal.log b/wandb/run-20241101_012438-1onnw7d5/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..e681b84e7de4de79abe7ed61d8f54d118f6fe20b --- /dev/null +++ b/wandb/run-20241101_012438-1onnw7d5/logs/debug-internal.log @@ -0,0 +1,16 @@ +{"time":"2024-11-01T01:24:38.174471309-04:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-11-01T01:24:38.174485399-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_012438-1onnw7d5/logs/debug-core.log"} +{"time":"2024-11-01T01:24:38.280498427-04:00","level":"INFO","msg":"created new stream","id":"1onnw7d5"} +{"time":"2024-11-01T01:24:38.280533768-04:00","level":"INFO","msg":"stream: started","id":"1onnw7d5"} +{"time":"2024-11-01T01:24:38.280568298-04:00","level":"INFO","msg":"sender: started","stream_id":"1onnw7d5"} +{"time":"2024-11-01T01:24:38.280549488-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"1onnw7d5"}} +{"time":"2024-11-01T01:24:38.280570018-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"1onnw7d5"}} +{"time":"2024-11-01T01:24:38.441821679-04:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-11-01T01:24:38.744246503-04:00","level":"INFO","msg":"stream: closing","id":"1onnw7d5"} +{"time":"2024-11-01T01:24:38.744301793-04:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2024-11-01T01:24:38.744928418-04:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2024-11-01T01:24:39.133873344-04:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2024-11-01T01:24:39.266150554-04:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"1onnw7d5"}} +{"time":"2024-11-01T01:24:39.266210184-04:00","level":"INFO","msg":"sender: closed","stream_id":"1onnw7d5"} +{"time":"2024-11-01T01:24:39.266202684-04:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"1onnw7d5"}} +{"time":"2024-11-01T01:24:39.266298455-04:00","level":"INFO","msg":"stream: closed","id":"1onnw7d5"} diff --git a/wandb/run-20241101_012438-1onnw7d5/logs/debug.log b/wandb/run-20241101_012438-1onnw7d5/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..5a4c1ba1a43343a046607ec84316f6b21f14ccdb --- /dev/null +++ b/wandb/run-20241101_012438-1onnw7d5/logs/debug.log @@ -0,0 +1,27 @@ +2024-11-01 01:24:38,170 INFO MainThread:676349 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-11-01 01:24:38,170 INFO MainThread:676349 [wandb_setup.py:_flush():79] Configure stats pid to 676349 +2024-11-01 01:24:38,170 INFO MainThread:676349 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-11-01 01:24:38,170 INFO MainThread:676349 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-11-01 01:24:38,170 INFO MainThread:676349 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-01 01:24:38,170 INFO MainThread:676349 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-01 01:24:38,170 INFO MainThread:676349 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'} +2024-11-01 01:24:38,170 INFO MainThread:676349 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-01 01:24:38,170 INFO MainThread:676349 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_012438-1onnw7d5/logs/debug.log +2024-11-01 01:24:38,170 INFO MainThread:676349 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_012438-1onnw7d5/logs/debug-internal.log +2024-11-01 01:24:38,170 INFO MainThread:676349 [wandb_init.py:init():621] calling init triggers +2024-11-01 01:24:38,170 INFO MainThread:676349 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-11-01 01:24:38,170 INFO MainThread:676349 [wandb_init.py:init():671] starting backend +2024-11-01 01:24:38,170 INFO MainThread:676349 [wandb_init.py:init():675] sending inform_init request +2024-11-01 01:24:38,171 INFO MainThread:676349 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-01 01:24:38,171 INFO MainThread:676349 [wandb_init.py:init():688] backend started and connected +2024-11-01 01:24:38,176 INFO MainThread:676349 [wandb_init.py:init():783] updated telemetry +2024-11-01 01:24:38,210 INFO MainThread:676349 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-11-01 01:24:38,438 INFO MainThread:676349 [wandb_init.py:init():867] starting run threads in backend +2024-11-01 01:24:38,576 INFO MainThread:676349 [wandb_run.py:_console_start():2463] atexit reg +2024-11-01 01:24:38,576 INFO MainThread:676349 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-11-01 01:24:38,576 INFO MainThread:676349 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-11-01 01:24:38,576 INFO MainThread:676349 [wandb_run.py:_redirect():2401] Redirects installed. +2024-11-01 01:24:38,579 INFO MainThread:676349 [wandb_init.py:init():911] run started, returning control to user process +2024-11-01 01:24:38,579 INFO MainThread:676349 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_nodeterministic', 'train_set': '10M', 'batch_size': 3, 'epoch': 6, 'seed': 0, 'lr': 5e-06} +2024-11-01 01:24:38,744 WARNING MsgRouterThr:676349 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20241101_012438-1onnw7d5/run-1onnw7d5.wandb b/wandb/run-20241101_012438-1onnw7d5/run-1onnw7d5.wandb new file mode 100644 index 0000000000000000000000000000000000000000..e621bcb2747e237afc75322063615d668bd64d2b Binary files /dev/null and b/wandb/run-20241101_012438-1onnw7d5/run-1onnw7d5.wandb differ diff --git a/wandb/run-20241101_012612-q08jbqqf/files/output.log b/wandb/run-20241101_012612-q08jbqqf/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..5824b3c1b0e7410256b3374a200db5bd3cc11c9d --- /dev/null +++ b/wandb/run-20241101_012612-q08jbqqf/files/output.log @@ -0,0 +1,12 @@ +Traceback (most recent call last): + File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 164, in + dataset = load_dataset('babylm_dataset_test.py', name=dataset_name, trust_remote_code=True) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/load.py", line 2074, in load_dataset + builder_instance = load_dataset_builder( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/load.py", line 1832, in load_dataset_builder + builder_instance: DatasetBuilder = builder_cls( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/builder.py", line 342, in __init__ + self.config, self.config_id = self._create_builder_config( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/builder.py", line 569, in _create_builder_config + raise ValueError( +ValueError: BuilderConfig 'babylm_shuffle_nodeterministic_10M_seed0' not found. Available: ['babylm_hop_control_10M_seed0', 'babylm_hop_tokens4_10M_seed0', 'babylm_hop_words4_10M_seed0', 'babylm_reverse_control_10M_seed0', 'babylm_reverse_partial_10M_seed0', 'babylm_reverse_full_10M_seed0', 'babylm_shuffle_control_10M_seed0', 'babylm_shuffle_nondeterministic_10M_seed0', 'babylm_shuffle_deterministic21_10M_seed0', 'babylm_shuffle_deterministic57_10M_seed0', 'babylm_shuffle_deterministic84_10M_seed0', 'babylm_shuffle_local3_10M_seed0', 'babylm_shuffle_local5_10M_seed0', 'babylm_shuffle_local10_10M_seed0', 'babylm_shuffle_even_odd_10M_seed0'] diff --git a/wandb/run-20241101_012612-q08jbqqf/run-q08jbqqf.wandb b/wandb/run-20241101_012612-q08jbqqf/run-q08jbqqf.wandb new file mode 100644 index 0000000000000000000000000000000000000000..68daa25a817e356805f333007f9f9b4102c1bb4d Binary files /dev/null and b/wandb/run-20241101_012612-q08jbqqf/run-q08jbqqf.wandb differ diff --git a/wandb/run-20241101_012613-k6o0lha8/files/config.yaml b/wandb/run-20241101_012613-k6o0lha8/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16d25ee0e4092474360045faaf0959cfc1b3e91d --- /dev/null +++ b/wandb/run-20241101_012613-k6o0lha8/files/config.yaml @@ -0,0 +1,49 @@ +_wandb: + value: + cli_version: 0.18.5 + m: [] + python_version: 3.9.19 + t: + "1": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "2": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "3": + - 13 + - 23 + - 55 + "4": 3.9.19 + "5": 0.18.5 + "6": 4.45.1 + "8": + - 5 + "12": 0.18.5 + "13": linux-x86_64 +batch_size: + value: 3 +epoch: + value: 6 +lr: + value: 5e-06 +perturbation: + value: shuffle_nodeterministic +seed: + value: 0 +train_set: + value: 10M diff --git a/wandb/run-20241101_012613-k6o0lha8/files/wandb-summary.json b/wandb/run-20241101_012613-k6o0lha8/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..6c37fe1cbbb8aed86fd461a79642cb991e4d35cf --- /dev/null +++ b/wandb/run-20241101_012613-k6o0lha8/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":0}} \ No newline at end of file diff --git a/wandb/run-20241101_012613-k6o0lha8/logs/debug-internal.log b/wandb/run-20241101_012613-k6o0lha8/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..2ddaa5e9051825c6537fc04c702d7454e56cb62d --- /dev/null +++ b/wandb/run-20241101_012613-k6o0lha8/logs/debug-internal.log @@ -0,0 +1,16 @@ +{"time":"2024-11-01T01:26:13.053618839-04:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-11-01T01:26:13.053632699-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_012613-k6o0lha8/logs/debug-core.log"} +{"time":"2024-11-01T01:26:13.161223586-04:00","level":"INFO","msg":"created new stream","id":"k6o0lha8"} +{"time":"2024-11-01T01:26:13.161267266-04:00","level":"INFO","msg":"stream: started","id":"k6o0lha8"} +{"time":"2024-11-01T01:26:13.161303247-04:00","level":"INFO","msg":"sender: started","stream_id":"k6o0lha8"} +{"time":"2024-11-01T01:26:13.161289557-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"k6o0lha8"}} +{"time":"2024-11-01T01:26:13.161370687-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"k6o0lha8"}} +{"time":"2024-11-01T01:26:13.366986845-04:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-11-01T01:26:13.481739202-04:00","level":"INFO","msg":"stream: closing","id":"k6o0lha8"} +{"time":"2024-11-01T01:26:13.481812843-04:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2024-11-01T01:26:13.482462557-04:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2024-11-01T01:26:14.147796598-04:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2024-11-01T01:26:14.269127141-04:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"k6o0lha8"}} +{"time":"2024-11-01T01:26:14.269155371-04:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"k6o0lha8"}} +{"time":"2024-11-01T01:26:14.269185711-04:00","level":"INFO","msg":"sender: closed","stream_id":"k6o0lha8"} +{"time":"2024-11-01T01:26:14.269277432-04:00","level":"INFO","msg":"stream: closed","id":"k6o0lha8"} diff --git a/wandb/run-20241101_012613-k6o0lha8/logs/debug.log b/wandb/run-20241101_012613-k6o0lha8/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..bf767d421e67818ae8668fb58f954498d021bd31 --- /dev/null +++ b/wandb/run-20241101_012613-k6o0lha8/logs/debug.log @@ -0,0 +1,27 @@ +2024-11-01 01:26:13,049 INFO MainThread:677635 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-11-01 01:26:13,049 INFO MainThread:677635 [wandb_setup.py:_flush():79] Configure stats pid to 677635 +2024-11-01 01:26:13,049 INFO MainThread:677635 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-11-01 01:26:13,049 INFO MainThread:677635 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-11-01 01:26:13,049 INFO MainThread:677635 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-01 01:26:13,049 INFO MainThread:677635 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-01 01:26:13,049 INFO MainThread:677635 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'} +2024-11-01 01:26:13,049 INFO MainThread:677635 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-01 01:26:13,049 INFO MainThread:677635 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_012613-k6o0lha8/logs/debug.log +2024-11-01 01:26:13,049 INFO MainThread:677635 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_012613-k6o0lha8/logs/debug-internal.log +2024-11-01 01:26:13,049 INFO MainThread:677635 [wandb_init.py:init():621] calling init triggers +2024-11-01 01:26:13,049 INFO MainThread:677635 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-11-01 01:26:13,049 INFO MainThread:677635 [wandb_init.py:init():671] starting backend +2024-11-01 01:26:13,049 INFO MainThread:677635 [wandb_init.py:init():675] sending inform_init request +2024-11-01 01:26:13,050 INFO MainThread:677635 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-01 01:26:13,051 INFO MainThread:677635 [wandb_init.py:init():688] backend started and connected +2024-11-01 01:26:13,054 INFO MainThread:677635 [wandb_init.py:init():783] updated telemetry +2024-11-01 01:26:13,084 INFO MainThread:677635 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-11-01 01:26:13,364 INFO MainThread:677635 [wandb_init.py:init():867] starting run threads in backend +2024-11-01 01:26:13,467 INFO MainThread:677635 [wandb_run.py:_console_start():2463] atexit reg +2024-11-01 01:26:13,467 INFO MainThread:677635 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-11-01 01:26:13,467 INFO MainThread:677635 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-11-01 01:26:13,467 INFO MainThread:677635 [wandb_run.py:_redirect():2401] Redirects installed. +2024-11-01 01:26:13,469 INFO MainThread:677635 [wandb_init.py:init():911] run started, returning control to user process +2024-11-01 01:26:13,469 INFO MainThread:677635 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_nodeterministic', 'train_set': '10M', 'batch_size': 3, 'epoch': 6, 'seed': 0, 'lr': 5e-06} +2024-11-01 01:26:13,481 WARNING MsgRouterThr:677635 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20241101_092804-qhsuxbxe/files/output.log b/wandb/run-20241101_092804-qhsuxbxe/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..eea515d23661a8a3eb41373060a435d81d7b5ba2 --- /dev/null +++ b/wandb/run-20241101_092804-qhsuxbxe/files/output.log @@ -0,0 +1,14 @@ +Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:14<00:00, 7.48s/it] +Map: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 17013/17013 [00:49<00:00, 346.42 examples/s] +tokenized_valid: Dataset({ + features: ['input_ids', 'attention_mask'], + num_rows: 600 +}) +/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +[2024-11-01 09:29:10,914] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-11-01 09:29:22,407] [INFO] [comm.py:652:init_distributed] cdb=None +Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination +Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root... +Loading extension module cpu_adam... +Time to load cpu_adam op: 4.957691431045532 seconds diff --git a/wandb/run-20241101_092804-qhsuxbxe/files/requirements.txt b/wandb/run-20241101_092804-qhsuxbxe/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..95a931302e269cc9e4fa5b719b6511f176ee2416 --- /dev/null +++ b/wandb/run-20241101_092804-qhsuxbxe/files/requirements.txt @@ -0,0 +1,147 @@ +funcsigs==1.0.2 +sentry-sdk==2.17.0 +multiprocess==0.70.16 +numpy==1.26.2 +pluralizer==1.2.0 +debugpy==1.6.7 +nvidia-cudnn-cu11==8.5.0.96 +deepspeed==0.15.2 +data==0.4 +pandas==2.1.3 +tomli==2.0.1 +charset-normalizer==3.3.2 +attrs==24.2.0 +aiosignal==1.3.1 +fsspec==2023.10.0 +nvidia-cusparse-cu11==11.7.4.91 +zipp==3.12.0 +mypy-extensions==1.0.0 +datasets==3.0.1 +joblib==1.3.2 +hjson==3.1.0 +traitlets==5.7.1 +stack-data==0.6.0 +transformers==4.45.1 +sympy==1.11.1 +Pygments==2.15.0 +docker-pycreds==0.4.0 +dill==0.3.8 +wheel==0.44.0 +prompt-toolkit==3.0.30 +parso==0.8.3 +ipykernel==6.23.1 +pyarrow==17.0.0 +certifi==2023.11.17 +nvidia-cufft-cu11==10.9.0.58 +six==1.16.0 +pydantic==2.9.2 +click==8.1.7 +nest-asyncio==1.5.6 +gmpy2==2.1.0 +matplotlib==3.8.2 +scipy==1.11.4 +typing_extensions==4.12.2 +statsmodels==0.14.0 +huggingface-hub==0.25.0 +frozenlist==1.4.1 +gpustat==1.1.1 +nvidia-nvtx-cu11==11.7.91 +safetensors==0.4.5 +stanza==1.9.2 +decorator==5.1.1 +seaborn==0.13.0 +sentencepiece==0.2.0 +PyYAML==6.0.1 +black==24.8.0 +protobuf==4.25.1 +pickleshare==0.7.5 +peft==0.13.0 +triton==2.0.0 +nvidia-cuda-runtime-cu11==11.7.99 +Jinja2==3.1.2 +nvidia-cusolver-cu11==11.4.0.1 +executing==1.2.0 +jupyter_client==8.1.0 +pluggy==1.3.0 +cmake==3.30.3 +pytz==2023.3.post1 +aiohappyeyeballs==2.4.2 +kiwisolver==1.4.5 +py-cpuinfo==9.0.0 +Pillow==10.1.0 +ptyprocess==0.7.0 +importlib_resources==6.4.5 +GitPython==3.1.43 +importlib-metadata==6.0.0 +iniconfig==2.0.0 +scikit-learn==1.3.2 +exceptiongroup==1.1.0 +networkx==2.8.6 +accelerate==1.0.0 +nltk==3.8.1 +shutilwhich==1.1.0 +fonttools==4.45.1 +future==0.18.3 +aiohttp==3.10.6 +wcwidth==0.2.5 +idna==3.6 +filelock==3.12.2 +pathspec==0.12.1 +jupyter_core==5.1.0 +lit==18.1.8 +nvidia-curand-cu11==10.2.10.91 +nvidia-cublas-cu11==11.10.3.66 +nvidia-ml-py==12.560.30 +msgpack==1.1.0 +python-dateutil==2.8.2 +blessed==1.20.0 +packaging==23.0 +gitdb==4.0.11 +yarl==1.13.0 +emoji==2.8.0 +tzdata==2023.3 +cycler==0.12.1 +tornado==6.2 +backcall==0.2.0 +plotnine==0.12.4 +ninja==1.11.1.1 +latex==0.7.0 +wandb==0.18.5 +setproctitle==1.3.3 +threadpoolctl==3.2.0 +requests==2.32.3 +pyparsing==3.1.1 +smmap==5.0.1 +pyzmq==23.0.0 +async-timeout==4.0.3 +annotated-types==0.7.0 +matplotlib-inline==0.1.6 +latexcodec==1.0.0 +ipython==8.0.0 +patsy==0.5.3 +contourpy==1.2.0 +multidict==6.1.0 +mizani==0.9.3 +urllib3==2.1.0 +tokenizers==0.20.0 +MarkupSafe==2.1.2 +pip==24.2 +pexpect==4.8.0 +tqdm==4.66.5 +jedi==0.18.2 +pydantic_core==2.23.4 +tempdir==0.7.1 +mpmath==1.2.1 +setuptools==72.1.0 +pytest==7.4.3 +pure-eval==0.2.2 +psutil==5.9.1 +comm==0.1.2 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +regex==2023.10.3 +platformdirs==2.5.2 +asttokens==2.2.1 +torch==2.0.0 +nvidia-nccl-cu11==2.14.3 +xxhash==3.5.0 diff --git a/wandb/run-20241101_092804-qhsuxbxe/files/wandb-metadata.json b/wandb/run-20241101_092804-qhsuxbxe/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..69561b5db125b5fba469069129829ca0b0882aea --- /dev/null +++ b/wandb/run-20241101_092804-qhsuxbxe/files/wandb-metadata.json @@ -0,0 +1,97 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-11-01T13:28:04.179365Z", + "args": [ + "--perturbation", + "shuffle_nondeterministic", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "6", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", + "codePath": "train/train_deep_wandb.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_deep_wandb.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA RTX A6000", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888559353856", + "used": "1754628071424" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + } + ], + "cudaVersion": "11.8" +} \ No newline at end of file diff --git a/wandb/run-20241101_092804-qhsuxbxe/logs/debug-internal.log b/wandb/run-20241101_092804-qhsuxbxe/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..13630e99a92d9160d1e2135ba001b5d8f0e09a51 --- /dev/null +++ b/wandb/run-20241101_092804-qhsuxbxe/logs/debug-internal.log @@ -0,0 +1,8 @@ +{"time":"2024-11-01T09:28:04.182358567-04:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-11-01T09:28:04.182375607-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_092804-qhsuxbxe/logs/debug-core.log"} +{"time":"2024-11-01T09:28:04.291337625-04:00","level":"INFO","msg":"created new stream","id":"qhsuxbxe"} +{"time":"2024-11-01T09:28:04.291389815-04:00","level":"INFO","msg":"stream: started","id":"qhsuxbxe"} +{"time":"2024-11-01T09:28:04.291442536-04:00","level":"INFO","msg":"sender: started","stream_id":"qhsuxbxe"} +{"time":"2024-11-01T09:28:04.291424526-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"qhsuxbxe"}} +{"time":"2024-11-01T09:28:04.291463006-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"qhsuxbxe"}} +{"time":"2024-11-01T09:28:04.534676015-04:00","level":"INFO","msg":"Starting system monitor"} diff --git a/wandb/run-20241101_092804-qhsuxbxe/logs/debug.log b/wandb/run-20241101_092804-qhsuxbxe/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b1387f867c5140979b3496aea55a6622c7aec17d --- /dev/null +++ b/wandb/run-20241101_092804-qhsuxbxe/logs/debug.log @@ -0,0 +1,26 @@ +2024-11-01 09:28:04,177 INFO MainThread:779816 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-11-01 09:28:04,177 INFO MainThread:779816 [wandb_setup.py:_flush():79] Configure stats pid to 779816 +2024-11-01 09:28:04,177 INFO MainThread:779816 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-11-01 09:28:04,177 INFO MainThread:779816 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-11-01 09:28:04,177 INFO MainThread:779816 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-01 09:28:04,177 INFO MainThread:779816 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-01 09:28:04,177 INFO MainThread:779816 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'} +2024-11-01 09:28:04,177 INFO MainThread:779816 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-01 09:28:04,177 INFO MainThread:779816 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_092804-qhsuxbxe/logs/debug.log +2024-11-01 09:28:04,177 INFO MainThread:779816 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_092804-qhsuxbxe/logs/debug-internal.log +2024-11-01 09:28:04,177 INFO MainThread:779816 [wandb_init.py:init():621] calling init triggers +2024-11-01 09:28:04,177 INFO MainThread:779816 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-11-01 09:28:04,177 INFO MainThread:779816 [wandb_init.py:init():671] starting backend +2024-11-01 09:28:04,177 INFO MainThread:779816 [wandb_init.py:init():675] sending inform_init request +2024-11-01 09:28:04,178 INFO MainThread:779816 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-01 09:28:04,179 INFO MainThread:779816 [wandb_init.py:init():688] backend started and connected +2024-11-01 09:28:04,182 INFO MainThread:779816 [wandb_init.py:init():783] updated telemetry +2024-11-01 09:28:04,218 INFO MainThread:779816 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-11-01 09:28:04,531 INFO MainThread:779816 [wandb_init.py:init():867] starting run threads in backend +2024-11-01 09:28:04,647 INFO MainThread:779816 [wandb_run.py:_console_start():2463] atexit reg +2024-11-01 09:28:04,647 INFO MainThread:779816 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-11-01 09:28:04,647 INFO MainThread:779816 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-11-01 09:28:04,647 INFO MainThread:779816 [wandb_run.py:_redirect():2401] Redirects installed. +2024-11-01 09:28:04,649 INFO MainThread:779816 [wandb_init.py:init():911] run started, returning control to user process +2024-11-01 09:28:04,649 INFO MainThread:779816 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_nondeterministic', 'train_set': '10M', 'batch_size': 3, 'epoch': 6, 'seed': 0, 'lr': 5e-06} diff --git a/wandb/run-20241101_201707-qparlnlu/files/output.log b/wandb/run-20241101_201707-qparlnlu/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..ece3ab4ce960469aa7029e1224c26f5a9daca93b --- /dev/null +++ b/wandb/run-20241101_201707-qparlnlu/files/output.log @@ -0,0 +1,16 @@ +Loading checkpoint shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:04<00:00, 2.21s/it] +tokenized_valid: Dataset({ + features: ['input_ids', 'attention_mask'], + num_rows: 600 +}) +/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +[2024-11-01 20:17:14,306] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-11-01 20:17:22,315] [INFO] [comm.py:652:init_distributed] cdb=None +Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination +Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root... +Emitting ninja build file /home/chunhui/.cache/torch_extensions/py39_cu117/cpu_adam/build.ninja... +Building extension module cpu_adam... +Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +Loading extension module cpu_adam... +Time to load cpu_adam op: 5.1879143714904785 seconds diff --git a/wandb/run-20241101_201707-qparlnlu/files/requirements.txt b/wandb/run-20241101_201707-qparlnlu/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..95a931302e269cc9e4fa5b719b6511f176ee2416 --- /dev/null +++ b/wandb/run-20241101_201707-qparlnlu/files/requirements.txt @@ -0,0 +1,147 @@ +funcsigs==1.0.2 +sentry-sdk==2.17.0 +multiprocess==0.70.16 +numpy==1.26.2 +pluralizer==1.2.0 +debugpy==1.6.7 +nvidia-cudnn-cu11==8.5.0.96 +deepspeed==0.15.2 +data==0.4 +pandas==2.1.3 +tomli==2.0.1 +charset-normalizer==3.3.2 +attrs==24.2.0 +aiosignal==1.3.1 +fsspec==2023.10.0 +nvidia-cusparse-cu11==11.7.4.91 +zipp==3.12.0 +mypy-extensions==1.0.0 +datasets==3.0.1 +joblib==1.3.2 +hjson==3.1.0 +traitlets==5.7.1 +stack-data==0.6.0 +transformers==4.45.1 +sympy==1.11.1 +Pygments==2.15.0 +docker-pycreds==0.4.0 +dill==0.3.8 +wheel==0.44.0 +prompt-toolkit==3.0.30 +parso==0.8.3 +ipykernel==6.23.1 +pyarrow==17.0.0 +certifi==2023.11.17 +nvidia-cufft-cu11==10.9.0.58 +six==1.16.0 +pydantic==2.9.2 +click==8.1.7 +nest-asyncio==1.5.6 +gmpy2==2.1.0 +matplotlib==3.8.2 +scipy==1.11.4 +typing_extensions==4.12.2 +statsmodels==0.14.0 +huggingface-hub==0.25.0 +frozenlist==1.4.1 +gpustat==1.1.1 +nvidia-nvtx-cu11==11.7.91 +safetensors==0.4.5 +stanza==1.9.2 +decorator==5.1.1 +seaborn==0.13.0 +sentencepiece==0.2.0 +PyYAML==6.0.1 +black==24.8.0 +protobuf==4.25.1 +pickleshare==0.7.5 +peft==0.13.0 +triton==2.0.0 +nvidia-cuda-runtime-cu11==11.7.99 +Jinja2==3.1.2 +nvidia-cusolver-cu11==11.4.0.1 +executing==1.2.0 +jupyter_client==8.1.0 +pluggy==1.3.0 +cmake==3.30.3 +pytz==2023.3.post1 +aiohappyeyeballs==2.4.2 +kiwisolver==1.4.5 +py-cpuinfo==9.0.0 +Pillow==10.1.0 +ptyprocess==0.7.0 +importlib_resources==6.4.5 +GitPython==3.1.43 +importlib-metadata==6.0.0 +iniconfig==2.0.0 +scikit-learn==1.3.2 +exceptiongroup==1.1.0 +networkx==2.8.6 +accelerate==1.0.0 +nltk==3.8.1 +shutilwhich==1.1.0 +fonttools==4.45.1 +future==0.18.3 +aiohttp==3.10.6 +wcwidth==0.2.5 +idna==3.6 +filelock==3.12.2 +pathspec==0.12.1 +jupyter_core==5.1.0 +lit==18.1.8 +nvidia-curand-cu11==10.2.10.91 +nvidia-cublas-cu11==11.10.3.66 +nvidia-ml-py==12.560.30 +msgpack==1.1.0 +python-dateutil==2.8.2 +blessed==1.20.0 +packaging==23.0 +gitdb==4.0.11 +yarl==1.13.0 +emoji==2.8.0 +tzdata==2023.3 +cycler==0.12.1 +tornado==6.2 +backcall==0.2.0 +plotnine==0.12.4 +ninja==1.11.1.1 +latex==0.7.0 +wandb==0.18.5 +setproctitle==1.3.3 +threadpoolctl==3.2.0 +requests==2.32.3 +pyparsing==3.1.1 +smmap==5.0.1 +pyzmq==23.0.0 +async-timeout==4.0.3 +annotated-types==0.7.0 +matplotlib-inline==0.1.6 +latexcodec==1.0.0 +ipython==8.0.0 +patsy==0.5.3 +contourpy==1.2.0 +multidict==6.1.0 +mizani==0.9.3 +urllib3==2.1.0 +tokenizers==0.20.0 +MarkupSafe==2.1.2 +pip==24.2 +pexpect==4.8.0 +tqdm==4.66.5 +jedi==0.18.2 +pydantic_core==2.23.4 +tempdir==0.7.1 +mpmath==1.2.1 +setuptools==72.1.0 +pytest==7.4.3 +pure-eval==0.2.2 +psutil==5.9.1 +comm==0.1.2 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +regex==2023.10.3 +platformdirs==2.5.2 +asttokens==2.2.1 +torch==2.0.0 +nvidia-nccl-cu11==2.14.3 +xxhash==3.5.0 diff --git a/wandb/run-20241101_201707-qparlnlu/files/wandb-metadata.json b/wandb/run-20241101_201707-qparlnlu/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..1c714c6241166477e7bed7e5048e057c63b362c2 --- /dev/null +++ b/wandb/run-20241101_201707-qparlnlu/files/wandb-metadata.json @@ -0,0 +1,97 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-11-02T00:17:07.873886Z", + "args": [ + "--perturbation", + "shuffle_nondeterministic", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "3", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", + "codePath": "train/train_deep_wandb.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_deep_wandb.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA RTX A6000", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888559353856", + "used": "1754802667520" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + } + ], + "cudaVersion": "11.8" +} \ No newline at end of file diff --git a/wandb/run-20241101_201707-qparlnlu/logs/debug-internal.log b/wandb/run-20241101_201707-qparlnlu/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..b319b34459ef53ca05d5f76c05bed16e8a618871 --- /dev/null +++ b/wandb/run-20241101_201707-qparlnlu/logs/debug-internal.log @@ -0,0 +1,8 @@ +{"time":"2024-11-01T20:17:07.876661786-04:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-11-01T20:17:07.876685067-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_201707-qparlnlu/logs/debug-core.log"} +{"time":"2024-11-01T20:17:07.984430937-04:00","level":"INFO","msg":"created new stream","id":"qparlnlu"} +{"time":"2024-11-01T20:17:07.984473777-04:00","level":"INFO","msg":"stream: started","id":"qparlnlu"} +{"time":"2024-11-01T20:17:07.984539197-04:00","level":"INFO","msg":"sender: started","stream_id":"qparlnlu"} +{"time":"2024-11-01T20:17:07.984500437-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"qparlnlu"}} +{"time":"2024-11-01T20:17:07.984517717-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"qparlnlu"}} +{"time":"2024-11-01T20:17:08.155922608-04:00","level":"INFO","msg":"Starting system monitor"} diff --git a/wandb/run-20241101_201707-qparlnlu/logs/debug.log b/wandb/run-20241101_201707-qparlnlu/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..0030f03ea5fe8825b31a63751d9ff8603dac3e6b --- /dev/null +++ b/wandb/run-20241101_201707-qparlnlu/logs/debug.log @@ -0,0 +1,26 @@ +2024-11-01 20:17:07,870 INFO MainThread:875623 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-11-01 20:17:07,870 INFO MainThread:875623 [wandb_setup.py:_flush():79] Configure stats pid to 875623 +2024-11-01 20:17:07,870 INFO MainThread:875623 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-11-01 20:17:07,871 INFO MainThread:875623 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-11-01 20:17:07,871 INFO MainThread:875623 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-01 20:17:07,871 INFO MainThread:875623 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-01 20:17:07,871 INFO MainThread:875623 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'} +2024-11-01 20:17:07,871 INFO MainThread:875623 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-01 20:17:07,871 INFO MainThread:875623 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_201707-qparlnlu/logs/debug.log +2024-11-01 20:17:07,871 INFO MainThread:875623 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_201707-qparlnlu/logs/debug-internal.log +2024-11-01 20:17:07,871 INFO MainThread:875623 [wandb_init.py:init():621] calling init triggers +2024-11-01 20:17:07,871 INFO MainThread:875623 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-11-01 20:17:07,871 INFO MainThread:875623 [wandb_init.py:init():671] starting backend +2024-11-01 20:17:07,871 INFO MainThread:875623 [wandb_init.py:init():675] sending inform_init request +2024-11-01 20:17:07,873 INFO MainThread:875623 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-01 20:17:07,873 INFO MainThread:875623 [wandb_init.py:init():688] backend started and connected +2024-11-01 20:17:07,877 INFO MainThread:875623 [wandb_init.py:init():783] updated telemetry +2024-11-01 20:17:07,900 INFO MainThread:875623 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-11-01 20:17:08,152 INFO MainThread:875623 [wandb_init.py:init():867] starting run threads in backend +2024-11-01 20:17:08,239 INFO MainThread:875623 [wandb_run.py:_console_start():2463] atexit reg +2024-11-01 20:17:08,239 INFO MainThread:875623 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-11-01 20:17:08,239 INFO MainThread:875623 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-11-01 20:17:08,239 INFO MainThread:875623 [wandb_run.py:_redirect():2401] Redirects installed. +2024-11-01 20:17:08,241 INFO MainThread:875623 [wandb_init.py:init():911] run started, returning control to user process +2024-11-01 20:17:08,241 INFO MainThread:875623 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_nondeterministic', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0, 'lr': 5e-06} diff --git a/wandb/run-20241105_161832-sl992h9i/files/output.log b/wandb/run-20241105_161832-sl992h9i/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..494098e6e4d0927970d2928a33121a676f27c4d1 --- /dev/null +++ b/wandb/run-20241105_161832-sl992h9i/files/output.log @@ -0,0 +1,28 @@ +Downloading shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [02:33<00:00, 76.52s/it] +Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:09<00:00, 4.70s/it] +Map: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 16425/16425 [00:45<00:00, 360.17 examples/s] +Map: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 17013/17013 [00:45<00:00, 371.27 examples/s] +tokenized_valid: Dataset({ + features: ['input_ids', 'attention_mask'], + num_rows: 1000 +}) +/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +[2024-11-05 16:23:11,215] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-11-05 16:23:22,612] [INFO] [comm.py:652:init_distributed] cdb=None +[2024-11-05 16:23:22,613] [INFO] [comm.py:683:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl +Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher. +Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination +Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root... +Loading extension module cpu_adam... +Time to load cpu_adam op: 31.5544273853302 seconds +wandb: WARNING The `run_name` is currently set to the same value as `TrainingArguments.output_dir`. If this was not intended, please specify a different run name by setting the `TrainingArguments.run_name` parameter. + 0%|▍ | 8/2736 [00:37<3:24:38, 4.50s/it] +{'loss': 3.0476, 'grad_norm': 0.0, 'learning_rate': 0.0, 'epoch': 0.0} +{'loss': 3.0289, 'grad_norm': 0.0, 'learning_rate': 0.0, 'epoch': 0.0} +{'loss': 2.9935, 'grad_norm': 0.0, 'learning_rate': 0.0, 'epoch': 0.0} +{'loss': 3.0139, 'grad_norm': 0.0, 'learning_rate': 0.0, 'epoch': 0.0} +{'loss': 2.9892, 'grad_norm': 0.0, 'learning_rate': 0.0, 'epoch': 0.01} +{'loss': 3.1252, 'grad_norm': 0.0, 'learning_rate': 0.0, 'epoch': 0.01} +{'loss': 3.0291, 'grad_norm': 0.0, 'learning_rate': 0.0, 'epoch': 0.01} +{'loss': 3.0607, 'grad_norm': 0.0, 'learning_rate': 0.0, 'epoch': 0.01} diff --git a/wandb/run-20241105_161832-sl992h9i/files/requirements.txt b/wandb/run-20241105_161832-sl992h9i/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..95a931302e269cc9e4fa5b719b6511f176ee2416 --- /dev/null +++ b/wandb/run-20241105_161832-sl992h9i/files/requirements.txt @@ -0,0 +1,147 @@ +funcsigs==1.0.2 +sentry-sdk==2.17.0 +multiprocess==0.70.16 +numpy==1.26.2 +pluralizer==1.2.0 +debugpy==1.6.7 +nvidia-cudnn-cu11==8.5.0.96 +deepspeed==0.15.2 +data==0.4 +pandas==2.1.3 +tomli==2.0.1 +charset-normalizer==3.3.2 +attrs==24.2.0 +aiosignal==1.3.1 +fsspec==2023.10.0 +nvidia-cusparse-cu11==11.7.4.91 +zipp==3.12.0 +mypy-extensions==1.0.0 +datasets==3.0.1 +joblib==1.3.2 +hjson==3.1.0 +traitlets==5.7.1 +stack-data==0.6.0 +transformers==4.45.1 +sympy==1.11.1 +Pygments==2.15.0 +docker-pycreds==0.4.0 +dill==0.3.8 +wheel==0.44.0 +prompt-toolkit==3.0.30 +parso==0.8.3 +ipykernel==6.23.1 +pyarrow==17.0.0 +certifi==2023.11.17 +nvidia-cufft-cu11==10.9.0.58 +six==1.16.0 +pydantic==2.9.2 +click==8.1.7 +nest-asyncio==1.5.6 +gmpy2==2.1.0 +matplotlib==3.8.2 +scipy==1.11.4 +typing_extensions==4.12.2 +statsmodels==0.14.0 +huggingface-hub==0.25.0 +frozenlist==1.4.1 +gpustat==1.1.1 +nvidia-nvtx-cu11==11.7.91 +safetensors==0.4.5 +stanza==1.9.2 +decorator==5.1.1 +seaborn==0.13.0 +sentencepiece==0.2.0 +PyYAML==6.0.1 +black==24.8.0 +protobuf==4.25.1 +pickleshare==0.7.5 +peft==0.13.0 +triton==2.0.0 +nvidia-cuda-runtime-cu11==11.7.99 +Jinja2==3.1.2 +nvidia-cusolver-cu11==11.4.0.1 +executing==1.2.0 +jupyter_client==8.1.0 +pluggy==1.3.0 +cmake==3.30.3 +pytz==2023.3.post1 +aiohappyeyeballs==2.4.2 +kiwisolver==1.4.5 +py-cpuinfo==9.0.0 +Pillow==10.1.0 +ptyprocess==0.7.0 +importlib_resources==6.4.5 +GitPython==3.1.43 +importlib-metadata==6.0.0 +iniconfig==2.0.0 +scikit-learn==1.3.2 +exceptiongroup==1.1.0 +networkx==2.8.6 +accelerate==1.0.0 +nltk==3.8.1 +shutilwhich==1.1.0 +fonttools==4.45.1 +future==0.18.3 +aiohttp==3.10.6 +wcwidth==0.2.5 +idna==3.6 +filelock==3.12.2 +pathspec==0.12.1 +jupyter_core==5.1.0 +lit==18.1.8 +nvidia-curand-cu11==10.2.10.91 +nvidia-cublas-cu11==11.10.3.66 +nvidia-ml-py==12.560.30 +msgpack==1.1.0 +python-dateutil==2.8.2 +blessed==1.20.0 +packaging==23.0 +gitdb==4.0.11 +yarl==1.13.0 +emoji==2.8.0 +tzdata==2023.3 +cycler==0.12.1 +tornado==6.2 +backcall==0.2.0 +plotnine==0.12.4 +ninja==1.11.1.1 +latex==0.7.0 +wandb==0.18.5 +setproctitle==1.3.3 +threadpoolctl==3.2.0 +requests==2.32.3 +pyparsing==3.1.1 +smmap==5.0.1 +pyzmq==23.0.0 +async-timeout==4.0.3 +annotated-types==0.7.0 +matplotlib-inline==0.1.6 +latexcodec==1.0.0 +ipython==8.0.0 +patsy==0.5.3 +contourpy==1.2.0 +multidict==6.1.0 +mizani==0.9.3 +urllib3==2.1.0 +tokenizers==0.20.0 +MarkupSafe==2.1.2 +pip==24.2 +pexpect==4.8.0 +tqdm==4.66.5 +jedi==0.18.2 +pydantic_core==2.23.4 +tempdir==0.7.1 +mpmath==1.2.1 +setuptools==72.1.0 +pytest==7.4.3 +pure-eval==0.2.2 +psutil==5.9.1 +comm==0.1.2 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +regex==2023.10.3 +platformdirs==2.5.2 +asttokens==2.2.1 +torch==2.0.0 +nvidia-nccl-cu11==2.14.3 +xxhash==3.5.0 diff --git a/wandb/run-20241105_161832-sl992h9i/files/wandb-metadata.json b/wandb/run-20241105_161832-sl992h9i/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..f130a0c6fcbc01c55de36e6ae691d384fb74e2b6 --- /dev/null +++ b/wandb/run-20241105_161832-sl992h9i/files/wandb-metadata.json @@ -0,0 +1,97 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-11-05T21:18:32.539569Z", + "args": [ + "--perturbation", + "shuffle_deterministic21", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "3", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", + "codePath": "train/train_deep_wandb.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_deep_wandb.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA RTX A6000", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888559353856", + "used": "1785200173056" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + } + ], + "cudaVersion": "11.8" +} \ No newline at end of file diff --git a/wandb/run-20241105_162824-9m8ga5sn/files/output.log b/wandb/run-20241105_162824-9m8ga5sn/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..7d67b7704d048346ee7bba583d224351eb7e65d6 --- /dev/null +++ b/wandb/run-20241105_162824-9m8ga5sn/files/output.log @@ -0,0 +1,39 @@ +Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:05<00:00, 2.69s/it] +Map: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 17013/17013 [00:47<00:00, 358.89 examples/s] +tokenized_valid: Dataset({ + features: ['input_ids', 'attention_mask'], + num_rows: 1000 +}) +/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +[2024-11-05 16:29:19,412] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-11-05 16:29:28,405] [INFO] [comm.py:652:init_distributed] cdb=None +Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination +Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root... +Loading extension module cpu_adam... +Time to load cpu_adam op: 5.117670059204102 seconds +Traceback (most recent call last): + File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 222, in + trainer.train() + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/trainer.py", line 2052, in train + return inner_training_loop( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/trainer.py", line 2204, in _inner_training_loop + model, self.optimizer = self.accelerator.prepare(self.model, self.optimizer) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/accelerate/accelerator.py", line 1344, in prepare + result = self._prepare_deepspeed(*args) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/accelerate/accelerator.py", line 1851, in _prepare_deepspeed + engine, optimizer, _, lr_scheduler = ds_initialize(**kwargs) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/__init__.py", line 193, in initialize + engine = DeepSpeedEngine(args=args, + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/runtime/engine.py", line 313, in __init__ + self._configure_optimizer(optimizer, model_parameters) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/runtime/engine.py", line 1302, in _configure_optimizer + self.optimizer = self._configure_zero_optimizer(basic_optimizer) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/runtime/engine.py", line 1560, in _configure_zero_optimizer + optimizer = DeepSpeedZeroOptimizer( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/runtime/zero/stage_1_and_2.py", line 326, in __init__ + param.cpu_data = param.data.cpu() +KeyboardInterrupt +Error in atexit._run_exitfuncs: +Traceback (most recent call last): + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/ops/transformer/inference/triton/matmul_ext.py", line 27, in is_nfs_path diff --git a/wandb/run-20241105_162824-9m8ga5sn/files/requirements.txt b/wandb/run-20241105_162824-9m8ga5sn/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..95a931302e269cc9e4fa5b719b6511f176ee2416 --- /dev/null +++ b/wandb/run-20241105_162824-9m8ga5sn/files/requirements.txt @@ -0,0 +1,147 @@ +funcsigs==1.0.2 +sentry-sdk==2.17.0 +multiprocess==0.70.16 +numpy==1.26.2 +pluralizer==1.2.0 +debugpy==1.6.7 +nvidia-cudnn-cu11==8.5.0.96 +deepspeed==0.15.2 +data==0.4 +pandas==2.1.3 +tomli==2.0.1 +charset-normalizer==3.3.2 +attrs==24.2.0 +aiosignal==1.3.1 +fsspec==2023.10.0 +nvidia-cusparse-cu11==11.7.4.91 +zipp==3.12.0 +mypy-extensions==1.0.0 +datasets==3.0.1 +joblib==1.3.2 +hjson==3.1.0 +traitlets==5.7.1 +stack-data==0.6.0 +transformers==4.45.1 +sympy==1.11.1 +Pygments==2.15.0 +docker-pycreds==0.4.0 +dill==0.3.8 +wheel==0.44.0 +prompt-toolkit==3.0.30 +parso==0.8.3 +ipykernel==6.23.1 +pyarrow==17.0.0 +certifi==2023.11.17 +nvidia-cufft-cu11==10.9.0.58 +six==1.16.0 +pydantic==2.9.2 +click==8.1.7 +nest-asyncio==1.5.6 +gmpy2==2.1.0 +matplotlib==3.8.2 +scipy==1.11.4 +typing_extensions==4.12.2 +statsmodels==0.14.0 +huggingface-hub==0.25.0 +frozenlist==1.4.1 +gpustat==1.1.1 +nvidia-nvtx-cu11==11.7.91 +safetensors==0.4.5 +stanza==1.9.2 +decorator==5.1.1 +seaborn==0.13.0 +sentencepiece==0.2.0 +PyYAML==6.0.1 +black==24.8.0 +protobuf==4.25.1 +pickleshare==0.7.5 +peft==0.13.0 +triton==2.0.0 +nvidia-cuda-runtime-cu11==11.7.99 +Jinja2==3.1.2 +nvidia-cusolver-cu11==11.4.0.1 +executing==1.2.0 +jupyter_client==8.1.0 +pluggy==1.3.0 +cmake==3.30.3 +pytz==2023.3.post1 +aiohappyeyeballs==2.4.2 +kiwisolver==1.4.5 +py-cpuinfo==9.0.0 +Pillow==10.1.0 +ptyprocess==0.7.0 +importlib_resources==6.4.5 +GitPython==3.1.43 +importlib-metadata==6.0.0 +iniconfig==2.0.0 +scikit-learn==1.3.2 +exceptiongroup==1.1.0 +networkx==2.8.6 +accelerate==1.0.0 +nltk==3.8.1 +shutilwhich==1.1.0 +fonttools==4.45.1 +future==0.18.3 +aiohttp==3.10.6 +wcwidth==0.2.5 +idna==3.6 +filelock==3.12.2 +pathspec==0.12.1 +jupyter_core==5.1.0 +lit==18.1.8 +nvidia-curand-cu11==10.2.10.91 +nvidia-cublas-cu11==11.10.3.66 +nvidia-ml-py==12.560.30 +msgpack==1.1.0 +python-dateutil==2.8.2 +blessed==1.20.0 +packaging==23.0 +gitdb==4.0.11 +yarl==1.13.0 +emoji==2.8.0 +tzdata==2023.3 +cycler==0.12.1 +tornado==6.2 +backcall==0.2.0 +plotnine==0.12.4 +ninja==1.11.1.1 +latex==0.7.0 +wandb==0.18.5 +setproctitle==1.3.3 +threadpoolctl==3.2.0 +requests==2.32.3 +pyparsing==3.1.1 +smmap==5.0.1 +pyzmq==23.0.0 +async-timeout==4.0.3 +annotated-types==0.7.0 +matplotlib-inline==0.1.6 +latexcodec==1.0.0 +ipython==8.0.0 +patsy==0.5.3 +contourpy==1.2.0 +multidict==6.1.0 +mizani==0.9.3 +urllib3==2.1.0 +tokenizers==0.20.0 +MarkupSafe==2.1.2 +pip==24.2 +pexpect==4.8.0 +tqdm==4.66.5 +jedi==0.18.2 +pydantic_core==2.23.4 +tempdir==0.7.1 +mpmath==1.2.1 +setuptools==72.1.0 +pytest==7.4.3 +pure-eval==0.2.2 +psutil==5.9.1 +comm==0.1.2 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +regex==2023.10.3 +platformdirs==2.5.2 +asttokens==2.2.1 +torch==2.0.0 +nvidia-nccl-cu11==2.14.3 +xxhash==3.5.0 diff --git a/wandb/run-20241105_162824-9m8ga5sn/files/wandb-metadata.json b/wandb/run-20241105_162824-9m8ga5sn/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..3a136b351ba4b8b628b54b7da9e5ab32698d9c3d --- /dev/null +++ b/wandb/run-20241105_162824-9m8ga5sn/files/wandb-metadata.json @@ -0,0 +1,97 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-11-05T21:28:24.367518Z", + "args": [ + "--perturbation", + "shuffle_deterministic21", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "3", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", + "codePath": "train/train_deep_wandb.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_deep_wandb.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA RTX A6000", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888559353856", + "used": "1785662992384" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + } + ], + "cudaVersion": "11.8" +} \ No newline at end of file diff --git a/wandb/run-20241105_162824-9m8ga5sn/logs/debug-internal.log b/wandb/run-20241105_162824-9m8ga5sn/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..9a00906852997f385fa599e3c8d8cebd1c6d7377 --- /dev/null +++ b/wandb/run-20241105_162824-9m8ga5sn/logs/debug-internal.log @@ -0,0 +1,8 @@ +{"time":"2024-11-05T16:28:24.371628326-05:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-11-05T16:28:24.371640476-05:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_162824-9m8ga5sn/logs/debug-core.log"} +{"time":"2024-11-05T16:28:24.485562338-05:00","level":"INFO","msg":"created new stream","id":"9m8ga5sn"} +{"time":"2024-11-05T16:28:24.485608159-05:00","level":"INFO","msg":"stream: started","id":"9m8ga5sn"} +{"time":"2024-11-05T16:28:24.485700019-05:00","level":"INFO","msg":"sender: started","stream_id":"9m8ga5sn"} +{"time":"2024-11-05T16:28:24.485694869-05:00","level":"INFO","msg":"handler: started","stream_id":{"value":"9m8ga5sn"}} +{"time":"2024-11-05T16:28:24.485686569-05:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"9m8ga5sn"}} +{"time":"2024-11-05T16:28:24.713271003-05:00","level":"INFO","msg":"Starting system monitor"} diff --git a/wandb/run-20241105_162824-9m8ga5sn/logs/debug.log b/wandb/run-20241105_162824-9m8ga5sn/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..1eff7f3981e0700a6a229eadce12f2037a12099e --- /dev/null +++ b/wandb/run-20241105_162824-9m8ga5sn/logs/debug.log @@ -0,0 +1,26 @@ +2024-11-05 16:28:24,365 INFO MainThread:1777857 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-11-05 16:28:24,365 INFO MainThread:1777857 [wandb_setup.py:_flush():79] Configure stats pid to 1777857 +2024-11-05 16:28:24,365 INFO MainThread:1777857 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-11-05 16:28:24,365 INFO MainThread:1777857 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-11-05 16:28:24,365 INFO MainThread:1777857 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-05 16:28:24,365 INFO MainThread:1777857 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-05 16:28:24,365 INFO MainThread:1777857 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'} +2024-11-05 16:28:24,365 INFO MainThread:1777857 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-05 16:28:24,365 INFO MainThread:1777857 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_162824-9m8ga5sn/logs/debug.log +2024-11-05 16:28:24,365 INFO MainThread:1777857 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_162824-9m8ga5sn/logs/debug-internal.log +2024-11-05 16:28:24,365 INFO MainThread:1777857 [wandb_init.py:init():621] calling init triggers +2024-11-05 16:28:24,365 INFO MainThread:1777857 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-11-05 16:28:24,365 INFO MainThread:1777857 [wandb_init.py:init():671] starting backend +2024-11-05 16:28:24,365 INFO MainThread:1777857 [wandb_init.py:init():675] sending inform_init request +2024-11-05 16:28:24,366 INFO MainThread:1777857 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-05 16:28:24,367 INFO MainThread:1777857 [wandb_init.py:init():688] backend started and connected +2024-11-05 16:28:24,370 INFO MainThread:1777857 [wandb_init.py:init():783] updated telemetry +2024-11-05 16:28:24,410 INFO MainThread:1777857 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-11-05 16:28:24,710 INFO MainThread:1777857 [wandb_init.py:init():867] starting run threads in backend +2024-11-05 16:28:24,797 INFO MainThread:1777857 [wandb_run.py:_console_start():2463] atexit reg +2024-11-05 16:28:24,797 INFO MainThread:1777857 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-11-05 16:28:24,797 INFO MainThread:1777857 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-11-05 16:28:24,798 INFO MainThread:1777857 [wandb_run.py:_redirect():2401] Redirects installed. +2024-11-05 16:28:24,799 INFO MainThread:1777857 [wandb_init.py:init():911] run started, returning control to user process +2024-11-05 16:28:24,799 INFO MainThread:1777857 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_deterministic21', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0, 'lr': 5e-06} diff --git a/wandb/run-20241105_162824-9m8ga5sn/run-9m8ga5sn.wandb b/wandb/run-20241105_162824-9m8ga5sn/run-9m8ga5sn.wandb new file mode 100644 index 0000000000000000000000000000000000000000..d560d977dafef80dcd22e447e2ff86a91da5ef6c Binary files /dev/null and b/wandb/run-20241105_162824-9m8ga5sn/run-9m8ga5sn.wandb differ diff --git a/wandb/run-20241105_163039-457mt3e1/files/config.yaml b/wandb/run-20241105_163039-457mt3e1/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba0e0eb5aa81d7186d91b3c8f342ad5574a4c100 --- /dev/null +++ b/wandb/run-20241105_163039-457mt3e1/files/config.yaml @@ -0,0 +1,49 @@ +_wandb: + value: + cli_version: 0.18.5 + m: [] + python_version: 3.9.19 + t: + "1": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "2": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "3": + - 13 + - 23 + - 55 + "4": 3.9.19 + "5": 0.18.5 + "6": 4.45.1 + "8": + - 5 + "12": 0.18.5 + "13": linux-x86_64 +batch_size: + value: 3 +epoch: + value: 3 +lr: + value: 5e-06 +perturbation: + value: shuffle_deterministic21 +seed: + value: 0 +train_set: + value: 10M diff --git a/wandb/run-20241105_163039-457mt3e1/files/output.log b/wandb/run-20241105_163039-457mt3e1/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..7c73c81bee25711b69a144d30644b0e24cb58adc --- /dev/null +++ b/wandb/run-20241105_163039-457mt3e1/files/output.log @@ -0,0 +1,68 @@ +model.safetensors.index.json: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████| 20.9k/20.9k [00:00<00:00, 8.35MB/s] +model-00001-of-00002.safetensors: 67%|███████████████████████████████████████████████████████████████████████▊ | 3.33G/4.97G [01:19<00:38, 42.0MB/s] +Downloading shards: 0%| | 0/2 [01:19 + model = AutoModelForCausalLM.from_pretrained(model_name, + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/models/auto/auto_factory.py", line 564, in from_pretrained + return model_class.from_pretrained( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/modeling_utils.py", line 3769, in from_pretrained + resolved_archive_file, sharded_metadata = get_checkpoint_shard_files( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/utils/hub.py", line 1098, in get_checkpoint_shard_files + cached_filename = cached_file( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/utils/hub.py", line 403, in cached_file + resolved_file = hf_hub_download( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/utils/_deprecation.py", line 101, in inner_f + return f(*args, **kwargs) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn + return fn(*args, **kwargs) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1232, in hf_hub_download + return _hf_hub_download_to_cache_dir( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1381, in _hf_hub_download_to_cache_dir + _download_to_tmp_and_move( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1915, in _download_to_tmp_and_move + http_get( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 541, in http_get + for chunk in r.iter_content(chunk_size=constants.DOWNLOAD_CHUNK_SIZE): + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/requests/models.py", line 820, in generate + yield from self.raw.stream(chunk_size, decode_content=True) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/urllib3/response.py", line 934, in stream + data = self.read(amt=amt, decode_content=decode_content) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/urllib3/response.py", line 877, in read + data = self._raw_read(amt) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/urllib3/response.py", line 833, in _raw_read + raise IncompleteRead(self._fp_bytes_read, self.length_remaining) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/contextlib.py", line 137, in __exit__ + self.gen.throw(typ, value, traceback) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/urllib3/response.py", line 748, in _error_catcher + self._connection.close() + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/urllib3/connection.py", line 272, in close + super().close() + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/http/client.py", line 965, in close + sock.close() # close it manually... there may be other refs + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/socket.py", line 501, in close + self._real_close() + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/ssl.py", line 1333, in _real_close + self._sslobj = None +KeyboardInterrupt diff --git a/wandb/run-20241105_163039-457mt3e1/files/requirements.txt b/wandb/run-20241105_163039-457mt3e1/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..95a931302e269cc9e4fa5b719b6511f176ee2416 --- /dev/null +++ b/wandb/run-20241105_163039-457mt3e1/files/requirements.txt @@ -0,0 +1,147 @@ +funcsigs==1.0.2 +sentry-sdk==2.17.0 +multiprocess==0.70.16 +numpy==1.26.2 +pluralizer==1.2.0 +debugpy==1.6.7 +nvidia-cudnn-cu11==8.5.0.96 +deepspeed==0.15.2 +data==0.4 +pandas==2.1.3 +tomli==2.0.1 +charset-normalizer==3.3.2 +attrs==24.2.0 +aiosignal==1.3.1 +fsspec==2023.10.0 +nvidia-cusparse-cu11==11.7.4.91 +zipp==3.12.0 +mypy-extensions==1.0.0 +datasets==3.0.1 +joblib==1.3.2 +hjson==3.1.0 +traitlets==5.7.1 +stack-data==0.6.0 +transformers==4.45.1 +sympy==1.11.1 +Pygments==2.15.0 +docker-pycreds==0.4.0 +dill==0.3.8 +wheel==0.44.0 +prompt-toolkit==3.0.30 +parso==0.8.3 +ipykernel==6.23.1 +pyarrow==17.0.0 +certifi==2023.11.17 +nvidia-cufft-cu11==10.9.0.58 +six==1.16.0 +pydantic==2.9.2 +click==8.1.7 +nest-asyncio==1.5.6 +gmpy2==2.1.0 +matplotlib==3.8.2 +scipy==1.11.4 +typing_extensions==4.12.2 +statsmodels==0.14.0 +huggingface-hub==0.25.0 +frozenlist==1.4.1 +gpustat==1.1.1 +nvidia-nvtx-cu11==11.7.91 +safetensors==0.4.5 +stanza==1.9.2 +decorator==5.1.1 +seaborn==0.13.0 +sentencepiece==0.2.0 +PyYAML==6.0.1 +black==24.8.0 +protobuf==4.25.1 +pickleshare==0.7.5 +peft==0.13.0 +triton==2.0.0 +nvidia-cuda-runtime-cu11==11.7.99 +Jinja2==3.1.2 +nvidia-cusolver-cu11==11.4.0.1 +executing==1.2.0 +jupyter_client==8.1.0 +pluggy==1.3.0 +cmake==3.30.3 +pytz==2023.3.post1 +aiohappyeyeballs==2.4.2 +kiwisolver==1.4.5 +py-cpuinfo==9.0.0 +Pillow==10.1.0 +ptyprocess==0.7.0 +importlib_resources==6.4.5 +GitPython==3.1.43 +importlib-metadata==6.0.0 +iniconfig==2.0.0 +scikit-learn==1.3.2 +exceptiongroup==1.1.0 +networkx==2.8.6 +accelerate==1.0.0 +nltk==3.8.1 +shutilwhich==1.1.0 +fonttools==4.45.1 +future==0.18.3 +aiohttp==3.10.6 +wcwidth==0.2.5 +idna==3.6 +filelock==3.12.2 +pathspec==0.12.1 +jupyter_core==5.1.0 +lit==18.1.8 +nvidia-curand-cu11==10.2.10.91 +nvidia-cublas-cu11==11.10.3.66 +nvidia-ml-py==12.560.30 +msgpack==1.1.0 +python-dateutil==2.8.2 +blessed==1.20.0 +packaging==23.0 +gitdb==4.0.11 +yarl==1.13.0 +emoji==2.8.0 +tzdata==2023.3 +cycler==0.12.1 +tornado==6.2 +backcall==0.2.0 +plotnine==0.12.4 +ninja==1.11.1.1 +latex==0.7.0 +wandb==0.18.5 +setproctitle==1.3.3 +threadpoolctl==3.2.0 +requests==2.32.3 +pyparsing==3.1.1 +smmap==5.0.1 +pyzmq==23.0.0 +async-timeout==4.0.3 +annotated-types==0.7.0 +matplotlib-inline==0.1.6 +latexcodec==1.0.0 +ipython==8.0.0 +patsy==0.5.3 +contourpy==1.2.0 +multidict==6.1.0 +mizani==0.9.3 +urllib3==2.1.0 +tokenizers==0.20.0 +MarkupSafe==2.1.2 +pip==24.2 +pexpect==4.8.0 +tqdm==4.66.5 +jedi==0.18.2 +pydantic_core==2.23.4 +tempdir==0.7.1 +mpmath==1.2.1 +setuptools==72.1.0 +pytest==7.4.3 +pure-eval==0.2.2 +psutil==5.9.1 +comm==0.1.2 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +regex==2023.10.3 +platformdirs==2.5.2 +asttokens==2.2.1 +torch==2.0.0 +nvidia-nccl-cu11==2.14.3 +xxhash==3.5.0 diff --git a/wandb/run-20241105_163039-457mt3e1/files/wandb-metadata.json b/wandb/run-20241105_163039-457mt3e1/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..6a98d725c0e3a00e85e4592e2801acff80ac1100 --- /dev/null +++ b/wandb/run-20241105_163039-457mt3e1/files/wandb-metadata.json @@ -0,0 +1,97 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-11-05T21:30:39.141776Z", + "args": [ + "--perturbation", + "shuffle_deterministic21", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "3", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", + "codePath": "train/train_deep_wandb.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_deep_wandb.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA RTX A6000", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888559353856", + "used": "1785984999424" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + } + ], + "cudaVersion": "11.8" +} \ No newline at end of file diff --git a/wandb/run-20241105_163039-457mt3e1/files/wandb-summary.json b/wandb/run-20241105_163039-457mt3e1/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..51cc9288ee1f562b22e175ccb65f3cee4534742c --- /dev/null +++ b/wandb/run-20241105_163039-457mt3e1/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":81}} \ No newline at end of file diff --git a/wandb/run-20241105_163039-457mt3e1/logs/debug-internal.log b/wandb/run-20241105_163039-457mt3e1/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..9d6b68ca5484a7fdede7b2631e545fbbdf832b66 --- /dev/null +++ b/wandb/run-20241105_163039-457mt3e1/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2024-11-05T16:30:39.144372085-05:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-11-05T16:30:39.144400915-05:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_163039-457mt3e1/logs/debug-core.log"} +{"time":"2024-11-05T16:30:39.251295098-05:00","level":"INFO","msg":"created new stream","id":"457mt3e1"} +{"time":"2024-11-05T16:30:39.251325458-05:00","level":"INFO","msg":"stream: started","id":"457mt3e1"} +{"time":"2024-11-05T16:30:39.251426298-05:00","level":"INFO","msg":"sender: started","stream_id":"457mt3e1"} +{"time":"2024-11-05T16:30:39.251417888-05:00","level":"INFO","msg":"handler: started","stream_id":{"value":"457mt3e1"}} +{"time":"2024-11-05T16:30:39.251388978-05:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"457mt3e1"}} +{"time":"2024-11-05T16:30:40.395953842-05:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-11-05T16:32:00.263761688-05:00","level":"INFO","msg":"stream: closing","id":"457mt3e1"} +{"time":"2024-11-05T16:32:00.263792138-05:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2024-11-05T16:32:00.264428021-05:00","level":"INFO","msg":"Stopped system monitor"} diff --git a/wandb/run-20241106_232725-8w3vgdka/files/config.yaml b/wandb/run-20241106_232725-8w3vgdka/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..45c9885a80c6c6551af0939982b4765da8541cf4 --- /dev/null +++ b/wandb/run-20241106_232725-8w3vgdka/files/config.yaml @@ -0,0 +1,49 @@ +_wandb: + value: + cli_version: 0.18.5 + m: [] + python_version: 3.9.19 + t: + "1": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "2": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "3": + - 13 + - 23 + - 55 + "4": 3.9.19 + "5": 0.18.5 + "6": 4.45.1 + "8": + - 5 + "12": 0.18.5 + "13": linux-x86_64 +batch_size: + value: 3 +epoch: + value: 3 +lr: + value: 5e-06 +perturbation: + value: shuffle_even_odd +seed: + value: 0 +train_set: + value: 10M diff --git a/wandb/run-20241106_232725-8w3vgdka/files/output.log b/wandb/run-20241106_232725-8w3vgdka/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..8b1af99393671560b017d0e75c103d768107d4a1 --- /dev/null +++ b/wandb/run-20241106_232725-8w3vgdka/files/output.log @@ -0,0 +1,60 @@ +Traceback (most recent call last): + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/utils/_http.py", line 406, in hf_raise_for_status + response.raise_for_status() + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/requests/models.py", line 1024, in raise_for_status + raise HTTPError(http_error_msg, response=self) +requests.exceptions.HTTPError: 401 Client Error: Unauthorized for url: https://huggingface.co/meta-llama/Llama-3.2-3B/resolve/main/config.json + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/utils/hub.py", line 403, in cached_file + resolved_file = hf_hub_download( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/utils/_deprecation.py", line 101, in inner_f + return f(*args, **kwargs) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn + return fn(*args, **kwargs) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1232, in hf_hub_download + return _hf_hub_download_to_cache_dir( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1339, in _hf_hub_download_to_cache_dir + _raise_on_head_call_error(head_call_error, force_download, local_files_only) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1854, in _raise_on_head_call_error + raise head_call_error + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1746, in _get_metadata_or_catch_error + metadata = get_hf_file_metadata( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn + return fn(*args, **kwargs) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1666, in get_hf_file_metadata + r = _request_wrapper( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 364, in _request_wrapper + response = _request_wrapper( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 388, in _request_wrapper + hf_raise_for_status(response) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/utils/_http.py", line 423, in hf_raise_for_status + raise _format(GatedRepoError, message, response) from e +huggingface_hub.errors.GatedRepoError: 401 Client Error. (Request ID: Root=1-672c41c6-195a8a13629e90f935840546;28f832e0-fb47-465e-94fd-14ae5c0783de) + +Cannot access gated repo for url https://huggingface.co/meta-llama/Llama-3.2-3B/resolve/main/config.json. +Access to model meta-llama/Llama-3.2-3B is restricted. You must have access to it and be authenticated to access it. Please log in. + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 174, in + model = AutoModelForCausalLM.from_pretrained(model_name, + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/models/auto/auto_factory.py", line 526, in from_pretrained + config, kwargs = AutoConfig.from_pretrained( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/models/auto/configuration_auto.py", line 1006, in from_pretrained + config_dict, unused_kwargs = PretrainedConfig.get_config_dict(pretrained_model_name_or_path, **kwargs) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/configuration_utils.py", line 567, in get_config_dict + config_dict, kwargs = cls._get_config_dict(pretrained_model_name_or_path, **kwargs) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/configuration_utils.py", line 626, in _get_config_dict + resolved_config_file = cached_file( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/utils/hub.py", line 421, in cached_file + raise EnvironmentError( +OSError: You are trying to access a gated repo. +Make sure to have access to it at https://huggingface.co/meta-llama/Llama-3.2-3B. +401 Client Error. (Request ID: Root=1-672c41c6-195a8a13629e90f935840546;28f832e0-fb47-465e-94fd-14ae5c0783de) + +Cannot access gated repo for url https://huggingface.co/meta-llama/Llama-3.2-3B/resolve/main/config.json. +Access to model meta-llama/Llama-3.2-3B is restricted. You must have access to it and be authenticated to access it. Please log in. diff --git a/wandb/run-20241106_232725-8w3vgdka/files/wandb-metadata.json b/wandb/run-20241106_232725-8w3vgdka/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..4d975ff9e6b1b5511bb8e06e23f100f9fecea2d8 --- /dev/null +++ b/wandb/run-20241106_232725-8w3vgdka/files/wandb-metadata.json @@ -0,0 +1,97 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-11-07T04:27:25.741736Z", + "args": [ + "--perturbation", + "shuffle_even_odd", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "3", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", + "codePath": "train/train_deep_wandb.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_deep_wandb.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA RTX A6000", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888559353856", + "used": "1775367761920" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + } + ], + "cudaVersion": "11.8" +} \ No newline at end of file diff --git a/wandb/run-20241106_232725-8w3vgdka/files/wandb-summary.json b/wandb/run-20241106_232725-8w3vgdka/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..fea5e15dda6eb8b3ffcccab00b24d57fa587c95b --- /dev/null +++ b/wandb/run-20241106_232725-8w3vgdka/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":24}} \ No newline at end of file diff --git a/wandb/run-20241106_232725-8w3vgdka/logs/debug-internal.log b/wandb/run-20241106_232725-8w3vgdka/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..71909a1b8cecaf558ef6e7b0b42f72bb62816233 --- /dev/null +++ b/wandb/run-20241106_232725-8w3vgdka/logs/debug-internal.log @@ -0,0 +1,16 @@ +{"time":"2024-11-06T23:27:25.743330103-05:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-11-06T23:27:25.743353024-05:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241106_232725-8w3vgdka/logs/debug-core.log"} +{"time":"2024-11-06T23:27:25.857989777-05:00","level":"INFO","msg":"created new stream","id":"8w3vgdka"} +{"time":"2024-11-06T23:27:25.858024717-05:00","level":"INFO","msg":"stream: started","id":"8w3vgdka"} +{"time":"2024-11-06T23:27:25.858049738-05:00","level":"INFO","msg":"sender: started","stream_id":"8w3vgdka"} +{"time":"2024-11-06T23:27:25.858045718-05:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"8w3vgdka"}} +{"time":"2024-11-06T23:27:25.858055578-05:00","level":"INFO","msg":"handler: started","stream_id":{"value":"8w3vgdka"}} +{"time":"2024-11-06T23:27:26.174443566-05:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-11-06T23:27:50.192790252-05:00","level":"INFO","msg":"stream: closing","id":"8w3vgdka"} +{"time":"2024-11-06T23:27:50.192824082-05:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2024-11-06T23:27:50.193465688-05:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2024-11-06T23:27:50.545945697-05:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2024-11-06T23:27:50.667419173-05:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"8w3vgdka"}} +{"time":"2024-11-06T23:27:50.667464933-05:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"8w3vgdka"}} +{"time":"2024-11-06T23:27:50.667485904-05:00","level":"INFO","msg":"sender: closed","stream_id":"8w3vgdka"} +{"time":"2024-11-06T23:27:50.667596075-05:00","level":"INFO","msg":"stream: closed","id":"8w3vgdka"} diff --git a/wandb/run-20241106_232725-8w3vgdka/logs/debug.log b/wandb/run-20241106_232725-8w3vgdka/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..08f60a5c5466284d6d7a3899b7f43a43f7017a65 --- /dev/null +++ b/wandb/run-20241106_232725-8w3vgdka/logs/debug.log @@ -0,0 +1,27 @@ +2024-11-06 23:27:25,739 INFO MainThread:1993249 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-11-06 23:27:25,739 INFO MainThread:1993249 [wandb_setup.py:_flush():79] Configure stats pid to 1993249 +2024-11-06 23:27:25,739 INFO MainThread:1993249 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-11-06 23:27:25,739 INFO MainThread:1993249 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-11-06 23:27:25,739 INFO MainThread:1993249 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-06 23:27:25,739 INFO MainThread:1993249 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-06 23:27:25,739 INFO MainThread:1993249 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'} +2024-11-06 23:27:25,739 INFO MainThread:1993249 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-06 23:27:25,739 INFO MainThread:1993249 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241106_232725-8w3vgdka/logs/debug.log +2024-11-06 23:27:25,739 INFO MainThread:1993249 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241106_232725-8w3vgdka/logs/debug-internal.log +2024-11-06 23:27:25,739 INFO MainThread:1993249 [wandb_init.py:init():621] calling init triggers +2024-11-06 23:27:25,739 INFO MainThread:1993249 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-11-06 23:27:25,739 INFO MainThread:1993249 [wandb_init.py:init():671] starting backend +2024-11-06 23:27:25,739 INFO MainThread:1993249 [wandb_init.py:init():675] sending inform_init request +2024-11-06 23:27:25,741 INFO MainThread:1993249 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-06 23:27:25,741 INFO MainThread:1993249 [wandb_init.py:init():688] backend started and connected +2024-11-06 23:27:25,746 INFO MainThread:1993249 [wandb_init.py:init():783] updated telemetry +2024-11-06 23:27:25,776 INFO MainThread:1993249 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-11-06 23:27:26,171 INFO MainThread:1993249 [wandb_init.py:init():867] starting run threads in backend +2024-11-06 23:27:26,291 INFO MainThread:1993249 [wandb_run.py:_console_start():2463] atexit reg +2024-11-06 23:27:26,291 INFO MainThread:1993249 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-11-06 23:27:26,291 INFO MainThread:1993249 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-11-06 23:27:26,291 INFO MainThread:1993249 [wandb_run.py:_redirect():2401] Redirects installed. +2024-11-06 23:27:26,293 INFO MainThread:1993249 [wandb_init.py:init():911] run started, returning control to user process +2024-11-06 23:27:26,294 INFO MainThread:1993249 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_even_odd', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0, 'lr': 5e-06} +2024-11-06 23:27:50,192 WARNING MsgRouterThr:1993249 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20241106_232725-8w3vgdka/run-8w3vgdka.wandb b/wandb/run-20241106_232725-8w3vgdka/run-8w3vgdka.wandb new file mode 100644 index 0000000000000000000000000000000000000000..b4eb818a3b15c639035b96f88c29d8dfee6013c3 Binary files /dev/null and b/wandb/run-20241106_232725-8w3vgdka/run-8w3vgdka.wandb differ diff --git a/wandb/run-20241106_234111-v7pqfaqj/files/wandb-metadata.json b/wandb/run-20241106_234111-v7pqfaqj/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..370995c683bd7b0449576ad56220e651af8934d1 --- /dev/null +++ b/wandb/run-20241106_234111-v7pqfaqj/files/wandb-metadata.json @@ -0,0 +1,97 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-11-07T04:41:11.201813Z", + "args": [ + "--perturbation", + "shuffle_even_odd", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "3", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", + "codePath": "train/train_deep_wandb.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_deep_wandb.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA RTX A6000", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888559353856", + "used": "1775602077696" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + } + ], + "cudaVersion": "11.8" +} \ No newline at end of file diff --git a/wandb/run-20241106_234111-v7pqfaqj/files/wandb-summary.json b/wandb/run-20241106_234111-v7pqfaqj/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..3a5cc5b3ae0648ea49e5e4291459904fd3499dc1 --- /dev/null +++ b/wandb/run-20241106_234111-v7pqfaqj/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":111}} \ No newline at end of file diff --git a/wandb/run-20241106_234111-v7pqfaqj/run-v7pqfaqj.wandb b/wandb/run-20241106_234111-v7pqfaqj/run-v7pqfaqj.wandb new file mode 100644 index 0000000000000000000000000000000000000000..bd5c3bdd1a34938ba665427dbcaf4617d4918b27 Binary files /dev/null and b/wandb/run-20241106_234111-v7pqfaqj/run-v7pqfaqj.wandb differ diff --git a/wandb/run-20241107_160818-1kyoikt6/files/config.yaml b/wandb/run-20241107_160818-1kyoikt6/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..45c9885a80c6c6551af0939982b4765da8541cf4 --- /dev/null +++ b/wandb/run-20241107_160818-1kyoikt6/files/config.yaml @@ -0,0 +1,49 @@ +_wandb: + value: + cli_version: 0.18.5 + m: [] + python_version: 3.9.19 + t: + "1": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "2": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "3": + - 13 + - 23 + - 55 + "4": 3.9.19 + "5": 0.18.5 + "6": 4.45.1 + "8": + - 5 + "12": 0.18.5 + "13": linux-x86_64 +batch_size: + value: 3 +epoch: + value: 3 +lr: + value: 5e-06 +perturbation: + value: shuffle_even_odd +seed: + value: 0 +train_set: + value: 10M diff --git a/wandb/run-20241107_160818-1kyoikt6/files/output.log b/wandb/run-20241107_160818-1kyoikt6/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..2a1b3dfecb694cf1c2ff6ff54029313332bf6aef --- /dev/null +++ b/wandb/run-20241107_160818-1kyoikt6/files/output.log @@ -0,0 +1,35 @@ +Traceback (most recent call last): + File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 165, in + dataset = load_dataset('babylm_dataset_test.py', name=dataset_name, trust_remote_code=True) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/load.py", line 2074, in load_dataset + builder_instance = load_dataset_builder( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/load.py", line 1832, in load_dataset_builder + builder_instance: DatasetBuilder = builder_cls( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/builder.py", line 415, in __init__ + extend_dataset_builder_for_streaming(self) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/streaming.py", line 120, in extend_dataset_builder_for_streaming + extend_module_for_streaming(builder.__module__, download_config=download_config) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/streaming.py", line 103, in extend_module_for_streaming + patch_submodule(module, "scipy.io.loadmat", wrap_auth(xsio_loadmat), attrs=["__version__"]).start() + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/utils/patching.py", line 108, in start + self.__enter__() + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/utils/patching.py", line 57, in __enter__ + submodule = import_module(".".join(submodules[: i + 1])) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/importlib/__init__.py", line 127, in import_module + return _bootstrap._gcd_import(name[level:], package, level) + File "", line 1030, in _gcd_import + File "", line 1007, in _find_and_load + File "", line 986, in _find_and_load_unlocked + File "", line 680, in _load_unlocked + File "", line 850, in exec_module + File "", line 228, in _call_with_frames_removed + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/scipy/io/__init__.py", line 97, in + from .matlab import loadmat, savemat, whosmat + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/scipy/io/matlab/__init__.py", line 45, in + from ._mio import loadmat, savemat, whosmat + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/scipy/io/matlab/_mio.py", line 10, in + from ._mio5 import MatFile5Reader, MatFile5Writer + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/scipy/io/matlab/_mio5.py", line 92, in + from ._mio5_utils import VarReader5 + File "", line 398, in parent +KeyboardInterrupt diff --git a/wandb/run-20241107_160818-1kyoikt6/files/wandb-metadata.json b/wandb/run-20241107_160818-1kyoikt6/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..077ebfbfd5db4e7c5cd153c872cf9c89d8f27a20 --- /dev/null +++ b/wandb/run-20241107_160818-1kyoikt6/files/wandb-metadata.json @@ -0,0 +1,97 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-11-07T21:08:18.520258Z", + "args": [ + "--perturbation", + "shuffle_even_odd", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "3", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", + "codePath": "train/train_deep_wandb.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_deep_wandb.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA RTX A6000", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888559353856", + "used": "1742796255232" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + } + ], + "cudaVersion": "11.8" +} \ No newline at end of file diff --git a/wandb/run-20241107_160818-1kyoikt6/files/wandb-summary.json b/wandb/run-20241107_160818-1kyoikt6/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..6c37fe1cbbb8aed86fd461a79642cb991e4d35cf --- /dev/null +++ b/wandb/run-20241107_160818-1kyoikt6/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":0}} \ No newline at end of file diff --git a/wandb/run-20241107_160818-1kyoikt6/logs/debug-internal.log b/wandb/run-20241107_160818-1kyoikt6/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..c0169e838d936a4ccc3baf10e990dac80dc3e1ab --- /dev/null +++ b/wandb/run-20241107_160818-1kyoikt6/logs/debug-internal.log @@ -0,0 +1,16 @@ +{"time":"2024-11-07T16:08:18.521987039-05:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-11-07T16:08:18.521998869-05:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241107_160818-1kyoikt6/logs/debug-core.log"} +{"time":"2024-11-07T16:08:18.628836873-05:00","level":"INFO","msg":"created new stream","id":"1kyoikt6"} +{"time":"2024-11-07T16:08:18.628866853-05:00","level":"INFO","msg":"stream: started","id":"1kyoikt6"} +{"time":"2024-11-07T16:08:18.628923123-05:00","level":"INFO","msg":"sender: started","stream_id":"1kyoikt6"} +{"time":"2024-11-07T16:08:18.628902073-05:00","level":"INFO","msg":"handler: started","stream_id":{"value":"1kyoikt6"}} +{"time":"2024-11-07T16:08:18.628885163-05:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"1kyoikt6"}} +{"time":"2024-11-07T16:08:18.802173196-05:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-11-07T16:08:18.948135816-05:00","level":"INFO","msg":"stream: closing","id":"1kyoikt6"} +{"time":"2024-11-07T16:08:18.948161106-05:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2024-11-07T16:08:18.948543609-05:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2024-11-07T16:08:19.56089018-05:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2024-11-07T16:08:19.692498609-05:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"1kyoikt6"}} +{"time":"2024-11-07T16:08:19.692550539-05:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"1kyoikt6"}} +{"time":"2024-11-07T16:08:19.69261121-05:00","level":"INFO","msg":"sender: closed","stream_id":"1kyoikt6"} +{"time":"2024-11-07T16:08:19.69266674-05:00","level":"INFO","msg":"stream: closed","id":"1kyoikt6"} diff --git a/wandb/run-20241107_160818-1kyoikt6/logs/debug.log b/wandb/run-20241107_160818-1kyoikt6/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..c91497df08889c5ee95672ecc9806a718ad5604b --- /dev/null +++ b/wandb/run-20241107_160818-1kyoikt6/logs/debug.log @@ -0,0 +1,27 @@ +2024-11-07 16:08:18,517 INFO MainThread:2097269 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-11-07 16:08:18,517 INFO MainThread:2097269 [wandb_setup.py:_flush():79] Configure stats pid to 2097269 +2024-11-07 16:08:18,517 INFO MainThread:2097269 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-11-07 16:08:18,517 INFO MainThread:2097269 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-11-07 16:08:18,517 INFO MainThread:2097269 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-07 16:08:18,517 INFO MainThread:2097269 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-07 16:08:18,517 INFO MainThread:2097269 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'} +2024-11-07 16:08:18,517 INFO MainThread:2097269 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-07 16:08:18,517 INFO MainThread:2097269 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241107_160818-1kyoikt6/logs/debug.log +2024-11-07 16:08:18,517 INFO MainThread:2097269 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241107_160818-1kyoikt6/logs/debug-internal.log +2024-11-07 16:08:18,517 INFO MainThread:2097269 [wandb_init.py:init():621] calling init triggers +2024-11-07 16:08:18,518 INFO MainThread:2097269 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-11-07 16:08:18,518 INFO MainThread:2097269 [wandb_init.py:init():671] starting backend +2024-11-07 16:08:18,518 INFO MainThread:2097269 [wandb_init.py:init():675] sending inform_init request +2024-11-07 16:08:18,519 INFO MainThread:2097269 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-07 16:08:18,519 INFO MainThread:2097269 [wandb_init.py:init():688] backend started and connected +2024-11-07 16:08:18,523 INFO MainThread:2097269 [wandb_init.py:init():783] updated telemetry +2024-11-07 16:08:18,553 INFO MainThread:2097269 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-11-07 16:08:18,798 INFO MainThread:2097269 [wandb_init.py:init():867] starting run threads in backend +2024-11-07 16:08:18,917 INFO MainThread:2097269 [wandb_run.py:_console_start():2463] atexit reg +2024-11-07 16:08:18,917 INFO MainThread:2097269 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-11-07 16:08:18,917 INFO MainThread:2097269 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-11-07 16:08:18,917 INFO MainThread:2097269 [wandb_run.py:_redirect():2401] Redirects installed. +2024-11-07 16:08:18,919 INFO MainThread:2097269 [wandb_init.py:init():911] run started, returning control to user process +2024-11-07 16:08:18,919 INFO MainThread:2097269 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_even_odd', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0, 'lr': 5e-06} +2024-11-07 16:08:18,948 WARNING MsgRouterThr:2097269 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20241128_161638-uo9cs4db/files/config.yaml b/wandb/run-20241128_161638-uo9cs4db/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5feb2a98b43e2e1c45b273f0fd6ee4e8b6deba5d --- /dev/null +++ b/wandb/run-20241128_161638-uo9cs4db/files/config.yaml @@ -0,0 +1,50 @@ +_wandb: + value: + cli_version: 0.18.5 + m: [] + python_version: 3.9.19 + t: + "1": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "2": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "3": + - 2 + - 13 + - 23 + - 55 + "4": 3.9.19 + "5": 0.18.5 + "6": 4.45.1 + "8": + - 5 + "12": 0.18.5 + "13": linux-x86_64 +batch_size: + value: 3 +epoch: + value: 3 +lr: + value: 5e-06 +perturbation: + value: reverse_full +seed: + value: 0 +train_set: + value: 10M diff --git a/wandb/run-20241128_161638-uo9cs4db/files/output.log b/wandb/run-20241128_161638-uo9cs4db/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..590477c66c057627aa30a83ca84d69123e6d8a4b --- /dev/null +++ b/wandb/run-20241128_161638-uo9cs4db/files/output.log @@ -0,0 +1,14 @@ +Map: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 17519/17519 [00:48<00:00, 359.03 examples/s] +Map: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 18140/18140 [00:49<00:00, 368.27 examples/s] +tokenized_valid: Dataset({ + features: ['input_ids', 'attention_mask'], + num_rows: 1000 +}) +/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +[2024-11-28 16:19:19,316] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-11-28 16:19:24,890] [INFO] [comm.py:652:init_distributed] cdb=None +Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination +Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root... +Loading extension module cpu_adam... +Time to load cpu_adam op: 3.787038803100586 seconds diff --git a/wandb/run-20241128_161638-uo9cs4db/files/wandb-metadata.json b/wandb/run-20241128_161638-uo9cs4db/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..ef0846d5ce07b7936714d18f8093d9882391cca0 --- /dev/null +++ b/wandb/run-20241128_161638-uo9cs4db/files/wandb-metadata.json @@ -0,0 +1,97 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-11-28T21:16:38.584154Z", + "args": [ + "--perturbation", + "reverse_full", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "3", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_llama_1B.py", + "codePath": "train/train_llama_1B.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_llama_1B.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA RTX A6000", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888559353856", + "used": "1723122212864" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + } + ], + "cudaVersion": "11.8" +} \ No newline at end of file diff --git a/wandb/run-20241128_161638-uo9cs4db/logs/debug-internal.log b/wandb/run-20241128_161638-uo9cs4db/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..69b889ed660de11ebe13608bef0b62a12470a618 --- /dev/null +++ b/wandb/run-20241128_161638-uo9cs4db/logs/debug-internal.log @@ -0,0 +1,18 @@ +{"time":"2024-11-28T16:16:38.585604895-05:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-11-28T16:16:38.585622135-05:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241128_161638-uo9cs4db/logs/debug-core.log"} +{"time":"2024-11-28T16:16:38.691935007-05:00","level":"INFO","msg":"created new stream","id":"uo9cs4db"} +{"time":"2024-11-28T16:16:38.691965457-05:00","level":"INFO","msg":"stream: started","id":"uo9cs4db"} +{"time":"2024-11-28T16:16:38.692022407-05:00","level":"INFO","msg":"sender: started","stream_id":"uo9cs4db"} +{"time":"2024-11-28T16:16:38.691993137-05:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"uo9cs4db"}} +{"time":"2024-11-28T16:16:38.692026257-05:00","level":"INFO","msg":"handler: started","stream_id":{"value":"uo9cs4db"}} +{"time":"2024-11-28T16:16:38.854225948-05:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-11-29T02:10:39.230773065-05:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/yaning1001-dartmouth-college/exp-impo-reverse/uo9cs4db/file_stream"} +{"time":"2024-11-29T05:45:58.381329211-05:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2024-11-29T05:45:58.40511475-05:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2024-11-29T05:45:59.381899786-05:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"saving job artifact","runtime_seconds":0.862828245}],"total_operations":1}} +{"time":"2024-11-29T05:45:59.829739628-05:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2024-11-29T05:46:01.174844115-05:00","level":"INFO","msg":"stream: closing","id":"uo9cs4db"} +{"time":"2024-11-29T05:46:01.174963226-05:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"uo9cs4db"}} +{"time":"2024-11-29T05:46:01.175076827-05:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"uo9cs4db"}} +{"time":"2024-11-29T05:46:01.175128827-05:00","level":"INFO","msg":"sender: closed","stream_id":"uo9cs4db"} +{"time":"2024-11-29T05:46:01.17549648-05:00","level":"INFO","msg":"stream: closed","id":"uo9cs4db"} diff --git a/wandb/run-20241129_083813-1lx1o6j4/files/config.yaml b/wandb/run-20241129_083813-1lx1o6j4/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..164653596defa3a070943f5b9f7cd18e7f2d95cc --- /dev/null +++ b/wandb/run-20241129_083813-1lx1o6j4/files/config.yaml @@ -0,0 +1,50 @@ +_wandb: + value: + cli_version: 0.18.5 + m: [] + python_version: 3.9.19 + t: + "1": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "2": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "3": + - 2 + - 13 + - 23 + - 55 + "4": 3.9.19 + "5": 0.18.5 + "6": 4.45.1 + "8": + - 5 + "12": 0.18.5 + "13": linux-x86_64 +batch_size: + value: 3 +epoch: + value: 3 +lr: + value: 5e-06 +perturbation: + value: reverse_partial +seed: + value: 0 +train_set: + value: 10M diff --git a/wandb/run-20241129_083813-1lx1o6j4/files/output.log b/wandb/run-20241129_083813-1lx1o6j4/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..9d9078c9ab86ab0621e6dc6f29f9a808a6b1dc51 --- /dev/null +++ b/wandb/run-20241129_083813-1lx1o6j4/files/output.log @@ -0,0 +1,14 @@ +Map: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 17519/17519 [00:47<00:00, 367.97 examples/s] +Map: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 18140/18140 [00:49<00:00, 370.02 examples/s] +tokenized_valid: Dataset({ + features: ['input_ids', 'attention_mask'], + num_rows: 1000 +}) +/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +[2024-11-29 08:40:53,266] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-11-29 08:40:58,380] [INFO] [comm.py:652:init_distributed] cdb=None +Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination +Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root... +Loading extension module cpu_adam... +Time to load cpu_adam op: 3.625563383102417 seconds diff --git a/wandb/run-20241129_083813-1lx1o6j4/files/wandb-metadata.json b/wandb/run-20241129_083813-1lx1o6j4/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..6277964fcb96a63794a44dddc269572c6d0382ab --- /dev/null +++ b/wandb/run-20241129_083813-1lx1o6j4/files/wandb-metadata.json @@ -0,0 +1,97 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-11-29T13:38:13.533167Z", + "args": [ + "--perturbation", + "reverse_partial", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "3", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_llama_1B.py", + "codePath": "train/train_llama_1B.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_llama_1B.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA RTX A6000", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888559353856", + "used": "1723546361856" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + } + ], + "cudaVersion": "11.8" +} \ No newline at end of file diff --git a/wandb/run-20241129_083813-1lx1o6j4/files/wandb-summary.json b/wandb/run-20241129_083813-1lx1o6j4/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..b2bf49411cde56dff9d227ffe9579f0f4bad56f2 --- /dev/null +++ b/wandb/run-20241129_083813-1lx1o6j4/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":26530}} \ No newline at end of file diff --git a/wandb/run-20241129_083813-1lx1o6j4/logs/debug-internal.log b/wandb/run-20241129_083813-1lx1o6j4/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..54abe97a33df30294adbeeb9fa55945b4bef159c --- /dev/null +++ b/wandb/run-20241129_083813-1lx1o6j4/logs/debug-internal.log @@ -0,0 +1,17 @@ +{"time":"2024-11-29T08:38:13.536342898-05:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-11-29T08:38:13.536360238-05:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241129_083813-1lx1o6j4/logs/debug-core.log"} +{"time":"2024-11-29T08:38:13.644815755-05:00","level":"INFO","msg":"created new stream","id":"1lx1o6j4"} +{"time":"2024-11-29T08:38:13.644873025-05:00","level":"INFO","msg":"stream: started","id":"1lx1o6j4"} +{"time":"2024-11-29T08:38:13.644946845-05:00","level":"INFO","msg":"sender: started","stream_id":"1lx1o6j4"} +{"time":"2024-11-29T08:38:13.644926245-05:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"1lx1o6j4"}} +{"time":"2024-11-29T08:38:13.644975686-05:00","level":"INFO","msg":"handler: started","stream_id":{"value":"1lx1o6j4"}} +{"time":"2024-11-29T08:38:13.913897479-05:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-11-29T16:00:24.499750374-05:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2024-11-29T16:00:24.539185755-05:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2024-11-29T16:00:25.096706141-05:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2024-11-29T16:00:25.218461704-05:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2024-11-29T16:00:26.227946477-05:00","level":"INFO","msg":"stream: closing","id":"1lx1o6j4"} +{"time":"2024-11-29T16:00:26.227989557-05:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"1lx1o6j4"}} +{"time":"2024-11-29T16:00:26.228016397-05:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"1lx1o6j4"}} +{"time":"2024-11-29T16:00:26.228029207-05:00","level":"INFO","msg":"sender: closed","stream_id":"1lx1o6j4"} +{"time":"2024-11-29T16:00:26.228137747-05:00","level":"INFO","msg":"stream: closed","id":"1lx1o6j4"} diff --git a/wandb/run-20241129_083813-1lx1o6j4/logs/debug.log b/wandb/run-20241129_083813-1lx1o6j4/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..17f8d54b40df6a0adf848435d2b62e437ea918a0 --- /dev/null +++ b/wandb/run-20241129_083813-1lx1o6j4/logs/debug.log @@ -0,0 +1,33 @@ +2024-11-29 08:38:13,530 INFO MainThread:3190019 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-11-29 08:38:13,530 INFO MainThread:3190019 [wandb_setup.py:_flush():79] Configure stats pid to 3190019 +2024-11-29 08:38:13,530 INFO MainThread:3190019 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-11-29 08:38:13,531 INFO MainThread:3190019 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-11-29 08:38:13,531 INFO MainThread:3190019 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-29 08:38:13,531 INFO MainThread:3190019 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-29 08:38:13,531 INFO MainThread:3190019 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_llama_1B.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_llama_1B.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_llama_1B.py'} +2024-11-29 08:38:13,531 INFO MainThread:3190019 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-29 08:38:13,531 INFO MainThread:3190019 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241129_083813-1lx1o6j4/logs/debug.log +2024-11-29 08:38:13,531 INFO MainThread:3190019 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241129_083813-1lx1o6j4/logs/debug-internal.log +2024-11-29 08:38:13,531 INFO MainThread:3190019 [wandb_init.py:init():621] calling init triggers +2024-11-29 08:38:13,531 INFO MainThread:3190019 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-11-29 08:38:13,531 INFO MainThread:3190019 [wandb_init.py:init():671] starting backend +2024-11-29 08:38:13,531 INFO MainThread:3190019 [wandb_init.py:init():675] sending inform_init request +2024-11-29 08:38:13,532 INFO MainThread:3190019 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-29 08:38:13,532 INFO MainThread:3190019 [wandb_init.py:init():688] backend started and connected +2024-11-29 08:38:13,537 INFO MainThread:3190019 [wandb_init.py:init():783] updated telemetry +2024-11-29 08:38:13,566 INFO MainThread:3190019 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-11-29 08:38:13,909 INFO MainThread:3190019 [wandb_init.py:init():867] starting run threads in backend +2024-11-29 08:38:14,001 INFO MainThread:3190019 [wandb_run.py:_console_start():2463] atexit reg +2024-11-29 08:38:14,001 INFO MainThread:3190019 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-11-29 08:38:14,001 INFO MainThread:3190019 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-11-29 08:38:14,001 INFO MainThread:3190019 [wandb_run.py:_redirect():2401] Redirects installed. +2024-11-29 08:38:14,003 INFO MainThread:3190019 [wandb_init.py:init():911] run started, returning control to user process +2024-11-29 08:38:14,004 INFO MainThread:3190019 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_partial', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0, 'lr': 5e-06} +2024-11-29 16:00:24,488 INFO MainThread:3190019 [wandb_run.py:_finish():2158] finishing run yaning1001-dartmouth-college/exp-impo-reverse/1lx1o6j4 +2024-11-29 16:00:24,498 INFO MainThread:3190019 [wandb_run.py:_atexit_cleanup():2426] got exitcode: 0 +2024-11-29 16:00:24,498 INFO MainThread:3190019 [wandb_run.py:_restore():2408] restore +2024-11-29 16:00:24,499 INFO MainThread:3190019 [wandb_run.py:_restore():2414] restore done +2024-11-29 16:00:26,222 INFO MainThread:3190019 [wandb_run.py:_footer_history_summary_info():3975] rendering history +2024-11-29 16:00:26,222 INFO MainThread:3190019 [wandb_run.py:_footer_history_summary_info():4007] rendering summary +2024-11-29 16:00:26,227 INFO MainThread:3190019 [wandb_run.py:_footer_sync_info():3934] logging synced files