Add files using upload-large-folder tool

wandb/run-20241030_010759-imp8g625/files/config.yaml

@@ -0,0 +1,47 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.9.19
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "3":
+                - 13
+                - 23
+                - 55
+            "4": 3.9.19
+            "5": 0.18.5
+            "6": 4.45.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+batch_size:
+    value: 3
+epoch:
+    value: 7
+perturbation:
+    value: reverse_control
+seed:
+    value: 0
+train_set:
+    value: 10M

wandb/run-20241030_010759-imp8g625/files/output.log

@@ -0,0 +1,4 @@
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 162, in <module>
+    dataset_name = f"babylm_{args.perturbation}_{args.train_zset}_seed{args.seed}"
+AttributeError: 'Namespace' object has no attribute 'train_zset'

wandb/run-20241030_013141-bkcoggdw/logs/debug.log

@@ -0,0 +1,27 @@
+2024-10-30 01:31:41,691 INFO    MainThread:335755 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-10-30 01:31:41,691 INFO    MainThread:335755 [wandb_setup.py:_flush():79] Configure stats pid to 335755
+2024-10-30 01:31:41,691 INFO    MainThread:335755 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-10-30 01:31:41,692 INFO    MainThread:335755 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-10-30 01:31:41,692 INFO    MainThread:335755 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-30 01:31:41,692 INFO    MainThread:335755 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-30 01:31:41,692 INFO    MainThread:335755 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-10-30 01:31:41,692 INFO    MainThread:335755 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-30 01:31:41,692 INFO    MainThread:335755 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_013141-bkcoggdw/logs/debug.log
+2024-10-30 01:31:41,692 INFO    MainThread:335755 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_013141-bkcoggdw/logs/debug-internal.log
+2024-10-30 01:31:41,692 INFO    MainThread:335755 [wandb_init.py:init():621] calling init triggers
+2024-10-30 01:31:41,692 INFO    MainThread:335755 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-10-30 01:31:41,692 INFO    MainThread:335755 [wandb_init.py:init():671] starting backend
+2024-10-30 01:31:41,692 INFO    MainThread:335755 [wandb_init.py:init():675] sending inform_init request
+2024-10-30 01:31:41,693 INFO    MainThread:335755 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-30 01:31:41,693 INFO    MainThread:335755 [wandb_init.py:init():688] backend started and connected
+2024-10-30 01:31:41,695 INFO    MainThread:335755 [wandb_init.py:init():783] updated telemetry
+2024-10-30 01:31:41,721 INFO    MainThread:335755 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-10-30 01:31:44,311 INFO    MainThread:335755 [wandb_init.py:init():867] starting run threads in backend
+2024-10-30 01:31:44,431 INFO    MainThread:335755 [wandb_run.py:_console_start():2463] atexit reg
+2024-10-30 01:31:44,431 INFO    MainThread:335755 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-10-30 01:31:44,431 INFO    MainThread:335755 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-10-30 01:31:44,431 INFO    MainThread:335755 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-10-30 01:31:44,432 INFO    MainThread:335755 [wandb_init.py:init():911] run started, returning control to user process
+2024-10-30 01:31:44,433 INFO    MainThread:335755 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_full', 'train_set': '10M', 'batch_size': 3, 'epoch': 7, 'seed': 0}
+2024-10-30 01:33:16,641 WARNING MsgRouterThr:335755 [router.py:message_loop():77] message_loop has been closed

wandb/run-20241031_001055-sr4xke8e/files/output.log

@@ -0,0 +1,13 @@
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:05<00:00,  2.82s/it]
+tokenized_valid: Dataset({
+    features: ['input_ids', 'attention_mask'],
+    num_rows: 600
+})
+/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
+  warnings.warn(
+[2024-10-31 00:11:03,358] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2024-10-31 00:11:12,356] [INFO] [comm.py:652:init_distributed] cdb=None
+Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination
+Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...
+Loading extension module cpu_adam...
+Time to load cpu_adam op: 5.375590562820435 seconds

wandb/run-20241031_001055-sr4xke8e/files/requirements.txt

@@ -0,0 +1,147 @@
+funcsigs==1.0.2
+sentry-sdk==2.17.0
+multiprocess==0.70.16
+numpy==1.26.2
+pluralizer==1.2.0
+debugpy==1.6.7
+nvidia-cudnn-cu11==8.5.0.96
+deepspeed==0.15.2
+data==0.4
+pandas==2.1.3
+tomli==2.0.1
+charset-normalizer==3.3.2
+attrs==24.2.0
+aiosignal==1.3.1
+fsspec==2023.10.0
+nvidia-cusparse-cu11==11.7.4.91
+zipp==3.12.0
+mypy-extensions==1.0.0
+datasets==3.0.1
+joblib==1.3.2
+hjson==3.1.0
+traitlets==5.7.1
+stack-data==0.6.0
+transformers==4.45.1
+sympy==1.11.1
+Pygments==2.15.0
+docker-pycreds==0.4.0
+dill==0.3.8
+wheel==0.44.0
+prompt-toolkit==3.0.30
+parso==0.8.3
+ipykernel==6.23.1
+pyarrow==17.0.0
+certifi==2023.11.17
+nvidia-cufft-cu11==10.9.0.58
+six==1.16.0
+pydantic==2.9.2
+click==8.1.7
+nest-asyncio==1.5.6
+gmpy2==2.1.0
+matplotlib==3.8.2
+scipy==1.11.4
+typing_extensions==4.12.2
+statsmodels==0.14.0
+huggingface-hub==0.25.0
+frozenlist==1.4.1
+gpustat==1.1.1
+nvidia-nvtx-cu11==11.7.91
+safetensors==0.4.5
+stanza==1.9.2
+decorator==5.1.1
+seaborn==0.13.0
+sentencepiece==0.2.0
+PyYAML==6.0.1
+black==24.8.0
+protobuf==4.25.1
+pickleshare==0.7.5
+peft==0.13.0
+triton==2.0.0
+nvidia-cuda-runtime-cu11==11.7.99
+Jinja2==3.1.2
+nvidia-cusolver-cu11==11.4.0.1
+executing==1.2.0
+jupyter_client==8.1.0
+pluggy==1.3.0
+cmake==3.30.3
+pytz==2023.3.post1
+aiohappyeyeballs==2.4.2
+kiwisolver==1.4.5
+py-cpuinfo==9.0.0
+Pillow==10.1.0
+ptyprocess==0.7.0
+importlib_resources==6.4.5
+GitPython==3.1.43
+importlib-metadata==6.0.0
+iniconfig==2.0.0
+scikit-learn==1.3.2
+exceptiongroup==1.1.0
+networkx==2.8.6
+accelerate==1.0.0
+nltk==3.8.1
+shutilwhich==1.1.0
+fonttools==4.45.1
+future==0.18.3
+aiohttp==3.10.6
+wcwidth==0.2.5
+idna==3.6
+filelock==3.12.2
+pathspec==0.12.1
+jupyter_core==5.1.0
+lit==18.1.8
+nvidia-curand-cu11==10.2.10.91
+nvidia-cublas-cu11==11.10.3.66
+nvidia-ml-py==12.560.30
+msgpack==1.1.0
+python-dateutil==2.8.2
+blessed==1.20.0
+packaging==23.0
+gitdb==4.0.11
+yarl==1.13.0
+emoji==2.8.0
+tzdata==2023.3
+cycler==0.12.1
+tornado==6.2
+backcall==0.2.0
+plotnine==0.12.4
+ninja==1.11.1.1
+latex==0.7.0
+wandb==0.18.5
+setproctitle==1.3.3
+threadpoolctl==3.2.0
+requests==2.32.3
+pyparsing==3.1.1
+smmap==5.0.1
+pyzmq==23.0.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+matplotlib-inline==0.1.6
+latexcodec==1.0.0
+ipython==8.0.0
+patsy==0.5.3
+contourpy==1.2.0
+multidict==6.1.0
+mizani==0.9.3
+urllib3==2.1.0
+tokenizers==0.20.0
+MarkupSafe==2.1.2
+pip==24.2
+pexpect==4.8.0
+tqdm==4.66.5
+jedi==0.18.2
+pydantic_core==2.23.4
+tempdir==0.7.1
+mpmath==1.2.1
+setuptools==72.1.0
+pytest==7.4.3
+pure-eval==0.2.2
+psutil==5.9.1
+comm==0.1.2
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+regex==2023.10.3
+platformdirs==2.5.2
+asttokens==2.2.1
+torch==2.0.0
+nvidia-nccl-cu11==2.14.3
+xxhash==3.5.0

wandb/run-20241031_001055-sr4xke8e/logs/debug.log

@@ -0,0 +1,26 @@
+2024-10-31 00:10:55,611 INFO    MainThread:479384 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-10-31 00:10:55,612 INFO    MainThread:479384 [wandb_setup.py:_flush():79] Configure stats pid to 479384
+2024-10-31 00:10:55,612 INFO    MainThread:479384 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-10-31 00:10:55,612 INFO    MainThread:479384 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-10-31 00:10:55,612 INFO    MainThread:479384 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-31 00:10:55,612 INFO    MainThread:479384 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-31 00:10:55,612 INFO    MainThread:479384 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-10-31 00:10:55,612 INFO    MainThread:479384 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-31 00:10:55,612 INFO    MainThread:479384 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_001055-sr4xke8e/logs/debug.log
+2024-10-31 00:10:55,612 INFO    MainThread:479384 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_001055-sr4xke8e/logs/debug-internal.log
+2024-10-31 00:10:55,612 INFO    MainThread:479384 [wandb_init.py:init():621] calling init triggers
+2024-10-31 00:10:55,612 INFO    MainThread:479384 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-10-31 00:10:55,612 INFO    MainThread:479384 [wandb_init.py:init():671] starting backend
+2024-10-31 00:10:55,612 INFO    MainThread:479384 [wandb_init.py:init():675] sending inform_init request
+2024-10-31 00:10:55,613 INFO    MainThread:479384 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-31 00:10:55,613 INFO    MainThread:479384 [wandb_init.py:init():688] backend started and connected
+2024-10-31 00:10:55,616 INFO    MainThread:479384 [wandb_init.py:init():783] updated telemetry
+2024-10-31 00:10:55,642 INFO    MainThread:479384 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-10-31 00:10:55,916 INFO    MainThread:479384 [wandb_init.py:init():867] starting run threads in backend
+2024-10-31 00:10:56,007 INFO    MainThread:479384 [wandb_run.py:_console_start():2463] atexit reg
+2024-10-31 00:10:56,007 INFO    MainThread:479384 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-10-31 00:10:56,007 INFO    MainThread:479384 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-10-31 00:10:56,008 INFO    MainThread:479384 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-10-31 00:10:56,010 INFO    MainThread:479384 [wandb_init.py:init():911] run started, returning control to user process
+2024-10-31 00:10:56,010 INFO    MainThread:479384 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_full', 'train_set': '10M', 'batch_size': 3, 'epoch': 6, 'seed': 0, 'lr': 1e-05}

wandb/run-20241031_114700-3cqkhntc/files/output.log

@@ -0,0 +1,220 @@
+model-00001-of-00002.safetensors: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████| 4.97G/4.97G [01:58<00:00, 42.0MB/s]
+model-00002-of-00002.safetensors: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1.46G/1.46G [00:34<00:00, 42.6MB/s]
+Downloading shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [02:32<00:00, 76.42s/it]▊| 1.46G/1.46G [00:34<00:00, 42.8MB/s]
+Loading checkpoint shards: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:04<00:00,  2.31s/it]
+generation_config.json: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 185/185 [00:00<00:00, 96.9kB/s]
+tokenized_valid: Dataset({
+    features: ['input_ids', 'attention_mask'],
+    num_rows: 600
+})
+/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
+  warnings.warn(
+[2024-10-31 11:49:40,282] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2024-10-31 11:49:48,785] [INFO] [comm.py:652:init_distributed] cdb=None
+[2024-10-31 11:49:48,785] [INFO] [comm.py:683:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl
+Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.
+Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination
+Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...
+Loading extension module cpu_adam...
+Time to load cpu_adam op: 4.844248533248901 seconds
+wandb: WARNING The `run_name` is currently set to the same value as `TrainingArguments.output_dir`. If this was not intended, please specify a different run name by setting the `TrainingArguments.run_name` parameter.
+                                                                                                                                                                                                                             
+{'loss': 2.7565, 'grad_norm': 0.0, 'learning_rate': 0.0001, 'epoch': 0.0}
+{'loss': 2.8049, 'grad_norm': 0.0, 'learning_rate': 0.0001, 'epoch': 0.0}
+{'loss': 2.7883, 'grad_norm': 0.0, 'learning_rate': 0.0001, 'epoch': 0.01}
+{'loss': 2.7843, 'grad_norm': 0.0, 'learning_rate': 0.0001, 'epoch': 0.01}
+{'loss': 2.7828, 'grad_norm': 0.0, 'learning_rate': 0.0001, 'epoch': 0.01}
+{'loss': 2.7493, 'grad_norm': 0.0, 'learning_rate': 0.0001, 'epoch': 0.01}
+{'loss': 2.7948, 'grad_norm': 0.0, 'learning_rate': 0.0001, 'epoch': 0.01}
+{'loss': 2.7614, 'grad_norm': 0.0, 'learning_rate': 0.0001, 'epoch': 0.02}
+{'loss': 2.7626, 'grad_norm': 0.0, 'learning_rate': 0.0001, 'epoch': 0.02}
+{'loss': 2.7527, 'grad_norm': 0.0, 'learning_rate': 0.0001, 'epoch': 0.02}
+                                                                                                                                                                                                                             
+{'eval_loss': 2.809370756149292, 'eval_runtime': 12.6957, 'eval_samples_per_second': 47.26, 'eval_steps_per_second': 1.024, 'epoch': 0.02}
+{'loss': 2.7977, 'grad_norm': 0.0, 'learning_rate': 0.0001, 'epoch': 0.02}
+{'loss': 2.7658, 'grad_norm': 0.0, 'learning_rate': 0.0001, 'epoch': 0.02}
+{'loss': 2.7787, 'grad_norm': 0.0, 'learning_rate': 0.0001, 'epoch': 0.03}
+{'loss': 2.7744, 'grad_norm': 0.0, 'learning_rate': 0.0001, 'epoch': 0.03}
+{'loss': 2.7916, 'grad_norm': 6.1797637939453125, 'learning_rate': 9.996577686516086e-05, 'epoch': 0.03}
+{'loss': 4.6579, 'grad_norm': 6.1797637939453125, 'learning_rate': 9.996577686516086e-05, 'epoch': 0.03}
+{'loss': 4.6189, 'grad_norm': 6.1797637939453125, 'learning_rate': 9.996577686516086e-05, 'epoch': 0.03}
+{'loss': 4.5916, 'grad_norm': 6.1797637939453125, 'learning_rate': 9.996577686516086e-05, 'epoch': 0.04}
+{'loss': 4.6312, 'grad_norm': 6.1797637939453125, 'learning_rate': 9.996577686516086e-05, 'epoch': 0.04}
+{'loss': 4.6558, 'grad_norm': 6.1797637939453125, 'learning_rate': 9.996577686516086e-05, 'epoch': 0.04}
+{'eval_loss': 4.709869861602783, 'eval_runtime': 11.5289, 'eval_samples_per_second': 52.043, 'eval_steps_per_second': 1.128, 'epoch': 0.04}
+{'loss': 4.6179, 'grad_norm': 85.80337524414062, 'learning_rate': 9.99315537303217e-05, 'epoch': 0.04}
+{'loss': 9.1321, 'grad_norm': 85.80337524414062, 'learning_rate': 9.99315537303217e-05, 'epoch': 0.05}
+{'loss': 9.0391, 'grad_norm': 85.80337524414062, 'learning_rate': 9.99315537303217e-05, 'epoch': 0.05}
+{'loss': 9.0784, 'grad_norm': 85.80337524414062, 'learning_rate': 9.99315537303217e-05, 'epoch': 0.05}
+{'loss': 9.0339, 'grad_norm': 340.5785827636719, 'learning_rate': 9.989733059548256e-05, 'epoch': 0.05}
+{'loss': 4.5744, 'grad_norm': 94.66028594970703, 'learning_rate': 9.98631074606434e-05, 'epoch': 0.05}
+{'loss': 3.428, 'grad_norm': 36.625179290771484, 'learning_rate': 9.982888432580426e-05, 'epoch': 0.06}
+{'loss': 2.8456, 'grad_norm': 16.80606460571289, 'learning_rate': 9.97946611909651e-05, 'epoch': 0.06}
+{'loss': 2.6428, 'grad_norm': 9.26777458190918, 'learning_rate': 9.976043805612595e-05, 'epoch': 0.06}
+{'loss': 2.6328, 'grad_norm': 9.187769889831543, 'learning_rate': 9.97262149212868e-05, 'epoch': 0.06}
+{'eval_loss': 2.725268602371216, 'eval_runtime': 11.5568, 'eval_samples_per_second': 51.918, 'eval_steps_per_second': 1.125, 'epoch': 0.06}
+{'loss': 2.5229, 'grad_norm': 6.6883864402771, 'learning_rate': 9.969199178644765e-05, 'epoch': 0.06}
+{'loss': 2.5127, 'grad_norm': 2.9667153358459473, 'learning_rate': 9.965776865160849e-05, 'epoch': 0.07}
+{'loss': 2.4111, 'grad_norm': 6.198004245758057, 'learning_rate': 9.962354551676935e-05, 'epoch': 0.07}
+{'loss': 2.4049, 'grad_norm': 2.4675652980804443, 'learning_rate': 9.958932238193019e-05, 'epoch': 0.07}
+{'loss': 2.4398, 'grad_norm': 3.319000720977783, 'learning_rate': 9.955509924709105e-05, 'epoch': 0.07}
+{'loss': 2.3516, 'grad_norm': 1.6354080438613892, 'learning_rate': 9.952087611225189e-05, 'epoch': 0.07}
+{'loss': 2.3761, 'grad_norm': 3.3287758827209473, 'learning_rate': 9.948665297741273e-05, 'epoch': 0.08}
+{'loss': 2.3453, 'grad_norm': 2.034022331237793, 'learning_rate': 9.945242984257359e-05, 'epoch': 0.08}
+{'loss': 2.3651, 'grad_norm': 2.3366262912750244, 'learning_rate': 9.941820670773443e-05, 'epoch': 0.08}
+{'loss': 2.3091, 'grad_norm': 2.418919801712036, 'learning_rate': 9.938398357289528e-05, 'epoch': 0.08}
+{'eval_loss': 2.5406363010406494, 'eval_runtime': 11.5689, 'eval_samples_per_second': 51.863, 'eval_steps_per_second': 1.124, 'epoch': 0.08}
+{'loss': 2.2928, 'grad_norm': 1.524388313293457, 'learning_rate': 9.934976043805613e-05, 'epoch': 0.08}
+{'loss': 2.2994, 'grad_norm': 3.2633249759674072, 'learning_rate': 9.931553730321698e-05, 'epoch': 0.09}
+{'loss': 2.2726, 'grad_norm': 1.2831145524978638, 'learning_rate': 9.928131416837782e-05, 'epoch': 0.09}
+{'loss': 2.2493, 'grad_norm': 2.7639758586883545, 'learning_rate': 9.924709103353868e-05, 'epoch': 0.09}
+{'loss': 2.2367, 'grad_norm': 1.9813640117645264, 'learning_rate': 9.921286789869952e-05, 'epoch': 0.09}
+{'loss': 2.2335, 'grad_norm': 1.9555699825286865, 'learning_rate': 9.917864476386038e-05, 'epoch': 0.09}
+{'loss': 2.2102, 'grad_norm': 2.355908155441284, 'learning_rate': 9.914442162902122e-05, 'epoch': 0.1}
+{'loss': 2.2115, 'grad_norm': 2.112484931945801, 'learning_rate': 9.911019849418207e-05, 'epoch': 0.1}
+{'loss': 2.1491, 'grad_norm': 1.4058802127838135, 'learning_rate': 9.907597535934292e-05, 'epoch': 0.1}
+{'loss': 2.1466, 'grad_norm': 2.0998988151550293, 'learning_rate': 9.904175222450377e-05, 'epoch': 0.1}
+{'eval_loss': 2.4130947589874268, 'eval_runtime': 11.5707, 'eval_samples_per_second': 51.855, 'eval_steps_per_second': 1.124, 'epoch': 0.1}
+{'loss': 2.1452, 'grad_norm': 1.3756946325302124, 'learning_rate': 9.900752908966461e-05, 'epoch': 0.1}
+{'loss': 2.1067, 'grad_norm': 1.696256399154663, 'learning_rate': 9.897330595482547e-05, 'epoch': 0.11}
+{'loss': 2.1575, 'grad_norm': 1.313455581665039, 'learning_rate': 9.893908281998631e-05, 'epoch': 0.11}
+{'loss': 2.1548, 'grad_norm': 1.4613757133483887, 'learning_rate': 9.890485968514717e-05, 'epoch': 0.11}
+{'loss': 2.0482, 'grad_norm': 1.3651756048202515, 'learning_rate': 9.887063655030801e-05, 'epoch': 0.11}
+{'loss': 2.0557, 'grad_norm': 0.923547089099884, 'learning_rate': 9.883641341546886e-05, 'epoch': 0.11}
+{'loss': 2.0637, 'grad_norm': 1.3289296627044678, 'learning_rate': 9.880219028062972e-05, 'epoch': 0.12}
+{'loss': 2.064, 'grad_norm': 1.094741940498352, 'learning_rate': 9.876796714579056e-05, 'epoch': 0.12}
+{'loss': 2.0745, 'grad_norm': 1.7548996210098267, 'learning_rate': 9.873374401095142e-05, 'epoch': 0.12}
+{'loss': 2.0973, 'grad_norm': 1.5266579389572144, 'learning_rate': 9.869952087611226e-05, 'epoch': 0.12}
+{'eval_loss': 2.3453779220581055, 'eval_runtime': 11.5705, 'eval_samples_per_second': 51.856, 'eval_steps_per_second': 1.124, 'epoch': 0.12}
+{'loss': 2.0411, 'grad_norm': 0.9586588144302368, 'learning_rate': 9.866529774127312e-05, 'epoch': 0.13}
+{'loss': 1.9945, 'grad_norm': 1.53722083568573, 'learning_rate': 9.863107460643394e-05, 'epoch': 0.13}
+{'loss': 2.0012, 'grad_norm': 1.1385856866836548, 'learning_rate': 9.85968514715948e-05, 'epoch': 0.13}
+{'loss': 1.9785, 'grad_norm': 1.4056907892227173, 'learning_rate': 9.856262833675564e-05, 'epoch': 0.13}
+{'loss': 2.0375, 'grad_norm': 1.1709682941436768, 'learning_rate': 9.85284052019165e-05, 'epoch': 0.13}
+{'loss': 1.9961, 'grad_norm': 1.3510781526565552, 'learning_rate': 9.849418206707734e-05, 'epoch': 0.14}
+{'loss': 1.9745, 'grad_norm': 1.6425883769989014, 'learning_rate': 9.84599589322382e-05, 'epoch': 0.14}
+{'loss': 1.9905, 'grad_norm': 1.2176721096038818, 'learning_rate': 9.842573579739904e-05, 'epoch': 0.14}
+{'loss': 1.9421, 'grad_norm': 1.3667473793029785, 'learning_rate': 9.839151266255989e-05, 'epoch': 0.14}
+{'loss': 1.9452, 'grad_norm': 1.8916258811950684, 'learning_rate': 9.835728952772075e-05, 'epoch': 0.14}
+{'eval_loss': 2.301532506942749, 'eval_runtime': 11.5625, 'eval_samples_per_second': 51.892, 'eval_steps_per_second': 1.124, 'epoch': 0.14}
+{'loss': 1.9262, 'grad_norm': 1.7627770900726318, 'learning_rate': 9.832306639288159e-05, 'epoch': 0.15}
+{'loss': 1.9736, 'grad_norm': 1.951368808746338, 'learning_rate': 9.828884325804245e-05, 'epoch': 0.15}
+{'loss': 1.8966, 'grad_norm': 2.173610210418701, 'learning_rate': 9.825462012320329e-05, 'epoch': 0.15}
+{'loss': 1.9133, 'grad_norm': 1.0655031204223633, 'learning_rate': 9.822039698836414e-05, 'epoch': 0.15}
+{'loss': 1.9703, 'grad_norm': 1.9031049013137817, 'learning_rate': 9.818617385352499e-05, 'epoch': 0.15}
+{'loss': 1.8632, 'grad_norm': 1.8678045272827148, 'learning_rate': 9.815195071868584e-05, 'epoch': 0.16}
+{'loss': 1.9308, 'grad_norm': 1.3070083856582642, 'learning_rate': 9.811772758384668e-05, 'epoch': 0.16}
+{'loss': 1.8469, 'grad_norm': 2.6512370109558105, 'learning_rate': 9.808350444900754e-05, 'epoch': 0.16}
+{'loss': 1.9147, 'grad_norm': 2.861356735229492, 'learning_rate': 9.804928131416838e-05, 'epoch': 0.16}
+{'loss': 1.8943, 'grad_norm': 1.9181678295135498, 'learning_rate': 9.801505817932924e-05, 'epoch': 0.16}
+{'eval_loss': 2.289658784866333, 'eval_runtime': 11.5613, 'eval_samples_per_second': 51.897, 'eval_steps_per_second': 1.124, 'epoch': 0.16}
+{'loss': 1.8777, 'grad_norm': 2.2638590335845947, 'learning_rate': 9.798083504449008e-05, 'epoch': 0.17}
+{'loss': 1.8578, 'grad_norm': 2.1869819164276123, 'learning_rate': 9.794661190965093e-05, 'epoch': 0.17}
+{'loss': 1.8953, 'grad_norm': 1.6545476913452148, 'learning_rate': 9.791238877481178e-05, 'epoch': 0.17}
+{'loss': 1.8593, 'grad_norm': 1.9989372491836548, 'learning_rate': 9.787816563997263e-05, 'epoch': 0.17}
+{'loss': 1.904, 'grad_norm': 1.3267912864685059, 'learning_rate': 9.784394250513347e-05, 'epoch': 0.17}
+{'loss': 1.8325, 'grad_norm': 1.8499799966812134, 'learning_rate': 9.780971937029433e-05, 'epoch': 0.18}
+{'loss': 1.7943, 'grad_norm': 1.453418254852295, 'learning_rate': 9.777549623545517e-05, 'epoch': 0.18}
+{'loss': 1.8924, 'grad_norm': 1.9336315393447876, 'learning_rate': 9.774127310061603e-05, 'epoch': 0.18}
+{'loss': 1.8241, 'grad_norm': 1.2739551067352295, 'learning_rate': 9.770704996577687e-05, 'epoch': 0.18}
+{'loss': 1.8327, 'grad_norm': 1.8302478790283203, 'learning_rate': 9.767282683093771e-05, 'epoch': 0.18}
+{'eval_loss': 2.255014181137085, 'eval_runtime': 11.5686, 'eval_samples_per_second': 51.865, 'eval_steps_per_second': 1.124, 'epoch': 0.18}
+{'loss': 1.8746, 'grad_norm': 1.3933634757995605, 'learning_rate': 9.763860369609857e-05, 'epoch': 0.19}
+{'loss': 1.8156, 'grad_norm': 1.53447425365448, 'learning_rate': 9.760438056125941e-05, 'epoch': 0.19}
+{'loss': 1.7668, 'grad_norm': 1.744211196899414, 'learning_rate': 9.757015742642026e-05, 'epoch': 0.19}
+{'loss': 1.8081, 'grad_norm': 1.784866213798523, 'learning_rate': 9.75359342915811e-05, 'epoch': 0.19}
+{'loss': 1.7898, 'grad_norm': 1.3280785083770752, 'learning_rate': 9.750171115674196e-05, 'epoch': 0.2}
+{'loss': 1.8135, 'grad_norm': 1.3295087814331055, 'learning_rate': 9.74674880219028e-05, 'epoch': 0.2}
+{'loss': 1.784, 'grad_norm': 1.3081759214401245, 'learning_rate': 9.743326488706366e-05, 'epoch': 0.2}
+{'loss': 1.7673, 'grad_norm': 1.2620002031326294, 'learning_rate': 9.73990417522245e-05, 'epoch': 0.2}
+{'loss': 1.7492, 'grad_norm': 1.2635165452957153, 'learning_rate': 9.736481861738536e-05, 'epoch': 0.2}
+{'loss': 1.8075, 'grad_norm': 1.3014883995056152, 'learning_rate': 9.73305954825462e-05, 'epoch': 0.21}
+{'eval_loss': 2.2504258155822754, 'eval_runtime': 11.5687, 'eval_samples_per_second': 51.864, 'eval_steps_per_second': 1.124, 'epoch': 0.21}
+{'loss': 1.7359, 'grad_norm': 1.090064287185669, 'learning_rate': 9.729637234770705e-05, 'epoch': 0.21}
+{'loss': 1.7055, 'grad_norm': 1.180134654045105, 'learning_rate': 9.726214921286791e-05, 'epoch': 0.21}
+{'loss': 1.7171, 'grad_norm': 1.392907738685608, 'learning_rate': 9.722792607802875e-05, 'epoch': 0.21}
+{'loss': 1.7232, 'grad_norm': 1.2521458864212036, 'learning_rate': 9.719370294318961e-05, 'epoch': 0.21}
+{'loss': 1.7408, 'grad_norm': 1.3325889110565186, 'learning_rate': 9.715947980835045e-05, 'epoch': 0.22}
+{'loss': 1.8078, 'grad_norm': 1.593698501586914, 'learning_rate': 9.71252566735113e-05, 'epoch': 0.22}
+{'loss': 1.7849, 'grad_norm': 1.251745343208313, 'learning_rate': 9.709103353867215e-05, 'epoch': 0.22}
+{'loss': 1.7266, 'grad_norm': 1.555539608001709, 'learning_rate': 9.7056810403833e-05, 'epoch': 0.22}
+{'loss': 1.7433, 'grad_norm': 1.5724241733551025, 'learning_rate': 9.702258726899384e-05, 'epoch': 0.22}
+{'loss': 1.7508, 'grad_norm': 1.3853074312210083, 'learning_rate': 9.69883641341547e-05, 'epoch': 0.23}
+{'eval_loss': 2.18611216545105, 'eval_runtime': 11.5643, 'eval_samples_per_second': 51.884, 'eval_steps_per_second': 1.124, 'epoch': 0.23}
+{'loss': 1.7278, 'grad_norm': 1.2098917961120605, 'learning_rate': 9.695414099931554e-05, 'epoch': 0.23}
+{'loss': 1.7654, 'grad_norm': 1.171714425086975, 'learning_rate': 9.69199178644764e-05, 'epoch': 0.23}
+{'loss': 1.741, 'grad_norm': 1.2655503749847412, 'learning_rate': 9.688569472963724e-05, 'epoch': 0.23}
+{'loss': 1.7532, 'grad_norm': 1.1870505809783936, 'learning_rate': 9.68514715947981e-05, 'epoch': 0.23}
+{'loss': 1.7446, 'grad_norm': 1.2463093996047974, 'learning_rate': 9.681724845995894e-05, 'epoch': 0.24}
+{'loss': 1.7164, 'grad_norm': 1.0629775524139404, 'learning_rate': 9.678302532511978e-05, 'epoch': 0.24}
+{'loss': 1.7579, 'grad_norm': 1.4846782684326172, 'learning_rate': 9.674880219028064e-05, 'epoch': 0.24}
+{'loss': 1.7606, 'grad_norm': 1.371062159538269, 'learning_rate': 9.671457905544148e-05, 'epoch': 0.24}
+{'loss': 1.7149, 'grad_norm': 1.347663164138794, 'learning_rate': 9.668035592060233e-05, 'epoch': 0.24}
+{'loss': 1.738, 'grad_norm': 1.2462127208709717, 'learning_rate': 9.664613278576318e-05, 'epoch': 0.25}
+{'eval_loss': 2.1583096981048584, 'eval_runtime': 11.5652, 'eval_samples_per_second': 51.88, 'eval_steps_per_second': 1.124, 'epoch': 0.25}
+{'loss': 1.7279, 'grad_norm': 1.3426861763000488, 'learning_rate': 9.661190965092403e-05, 'epoch': 0.25}
+{'loss': 1.6961, 'grad_norm': 1.3338221311569214, 'learning_rate': 9.657768651608487e-05, 'epoch': 0.25}
+{'loss': 1.6725, 'grad_norm': 1.252794623374939, 'learning_rate': 9.654346338124573e-05, 'epoch': 0.25}
+{'loss': 1.7193, 'grad_norm': 1.3079990148544312, 'learning_rate': 9.650924024640657e-05, 'epoch': 0.25}
+{'loss': 1.7285, 'grad_norm': 1.2246983051300049, 'learning_rate': 9.647501711156743e-05, 'epoch': 0.26}
+{'loss': 1.7116, 'grad_norm': 1.2157689332962036, 'learning_rate': 9.644079397672827e-05, 'epoch': 0.26}
+{'loss': 1.7175, 'grad_norm': 1.3069276809692383, 'learning_rate': 9.640657084188912e-05, 'epoch': 0.26}
+{'loss': 1.7227, 'grad_norm': 1.3949460983276367, 'learning_rate': 9.637234770704997e-05, 'epoch': 0.26}
+{'loss': 1.7351, 'grad_norm': 1.346846580505371, 'learning_rate': 9.633812457221082e-05, 'epoch': 0.26}
+{'loss': 1.7212, 'grad_norm': 1.3208940029144287, 'learning_rate': 9.630390143737166e-05, 'epoch': 0.27}
+{'eval_loss': 2.1235294342041016, 'eval_runtime': 11.5634, 'eval_samples_per_second': 51.888, 'eval_steps_per_second': 1.124, 'epoch': 0.27}
+{'loss': 1.6919, 'grad_norm': 1.1460075378417969, 'learning_rate': 9.626967830253252e-05, 'epoch': 0.27}
+{'loss': 1.6896, 'grad_norm': 1.5786269903182983, 'learning_rate': 9.623545516769336e-05, 'epoch': 0.27}
+{'loss': 1.7246, 'grad_norm': 1.4066743850708008, 'learning_rate': 9.620123203285422e-05, 'epoch': 0.27}
+{'loss': 1.6981, 'grad_norm': 1.4564768075942993, 'learning_rate': 9.616700889801506e-05, 'epoch': 0.28}
+{'loss': 1.6919, 'grad_norm': 1.3670514822006226, 'learning_rate': 9.613278576317591e-05, 'epoch': 0.28}
+{'loss': 1.7132, 'grad_norm': 1.290029525756836, 'learning_rate': 9.609856262833677e-05, 'epoch': 0.28}
+{'loss': 1.686, 'grad_norm': 1.171757459640503, 'learning_rate': 9.606433949349761e-05, 'epoch': 0.28}
+{'loss': 1.667, 'grad_norm': 1.3537696599960327, 'learning_rate': 9.603011635865847e-05, 'epoch': 0.28}
+{'loss': 1.704, 'grad_norm': 1.4693126678466797, 'learning_rate': 9.599589322381931e-05, 'epoch': 0.29}
+{'loss': 1.6503, 'grad_norm': 1.2005213499069214, 'learning_rate': 9.596167008898017e-05, 'epoch': 0.29}
+{'eval_loss': 2.1207213401794434, 'eval_runtime': 11.5721, 'eval_samples_per_second': 51.849, 'eval_steps_per_second': 1.123, 'epoch': 0.29}
+{'loss': 1.7115, 'grad_norm': 1.1947609186172485, 'learning_rate': 9.592744695414101e-05, 'epoch': 0.29}
+{'loss': 1.6809, 'grad_norm': 1.4202208518981934, 'learning_rate': 9.589322381930185e-05, 'epoch': 0.29}
+{'loss': 1.682, 'grad_norm': 1.5427730083465576, 'learning_rate': 9.585900068446269e-05, 'epoch': 0.29}
+{'loss': 1.6823, 'grad_norm': 1.2999160289764404, 'learning_rate': 9.582477754962355e-05, 'epoch': 0.3}
+{'loss': 1.6988, 'grad_norm': 1.2814061641693115, 'learning_rate': 9.579055441478439e-05, 'epoch': 0.3}
+{'loss': 1.6891, 'grad_norm': 1.311761736869812, 'learning_rate': 9.575633127994524e-05, 'epoch': 0.3}
+{'loss': 1.6682, 'grad_norm': 1.2029725313186646, 'learning_rate': 9.572210814510609e-05, 'epoch': 0.3}
+{'loss': 1.67, 'grad_norm': 1.0917092561721802, 'learning_rate': 9.568788501026694e-05, 'epoch': 0.3}
+{'loss': 1.6268, 'grad_norm': 1.335014820098877, 'learning_rate': 9.56536618754278e-05, 'epoch': 0.31}
+{'loss': 1.6501, 'grad_norm': 1.3213744163513184, 'learning_rate': 9.561943874058864e-05, 'epoch': 0.31}
+{'eval_loss': 2.103264570236206, 'eval_runtime': 11.5773, 'eval_samples_per_second': 51.826, 'eval_steps_per_second': 1.123, 'epoch': 0.31}
+{'loss': 1.6835, 'grad_norm': 1.2449109554290771, 'learning_rate': 9.55852156057495e-05, 'epoch': 0.31}
+{'loss': 1.6984, 'grad_norm': 1.4669255018234253, 'learning_rate': 9.555099247091034e-05, 'epoch': 0.31}
+{'loss': 1.6771, 'grad_norm': 1.2852448225021362, 'learning_rate': 9.551676933607119e-05, 'epoch': 0.31}
+{'loss': 1.6275, 'grad_norm': 1.3089197874069214, 'learning_rate': 9.548254620123203e-05, 'epoch': 0.32}
+{'loss': 1.6546, 'grad_norm': 1.453117847442627, 'learning_rate': 9.544832306639289e-05, 'epoch': 0.32}
+{'loss': 1.6131, 'grad_norm': 1.1836069822311401, 'learning_rate': 9.541409993155373e-05, 'epoch': 0.32}
+{'loss': 1.6603, 'grad_norm': 1.197521686553955, 'learning_rate': 9.537987679671459e-05, 'epoch': 0.32}
+{'loss': 1.6618, 'grad_norm': 1.3445478677749634, 'learning_rate': 9.534565366187543e-05, 'epoch': 0.32}
+{'loss': 1.6263, 'grad_norm': 1.3089582920074463, 'learning_rate': 9.531143052703629e-05, 'epoch': 0.33}
+    trainer.train()
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/trainer.py", line 2052, in train
+    return inner_training_loop(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/trainer.py", line 2388, in _inner_training_loop
+    tr_loss_step = self.training_step(model, inputs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/trainer.py", line 3518, in training_step
+    self.accelerator.backward(loss, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/accelerate/accelerator.py", line 2238, in backward
+    self.deepspeed_engine_wrapped.backward(loss, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/accelerate/utils/deepspeed.py", line 186, in backward
+    self.engine.backward(loss, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/utils/nvtx.py", line 18, in wrapped_fn
+    ret_val = func(*args, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/runtime/engine.py", line 2020, in backward
+    self.optimizer.backward(loss, retain_graph=retain_graph)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/runtime/zero/stage_1_and_2.py", line 2063, in backward
+    self.loss_scaler.backward(loss.float(), retain_graph=retain_graph)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/runtime/fp16/loss_scaler.py", line 63, in backward
+    scaled_loss.backward(retain_graph=retain_graph)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/torch/_tensor.py", line 487, in backward
+    torch.autograd.backward(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/torch/autograd/__init__.py", line 200, in backward
+    Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
+KeyboardInterrupt

wandb/run-20241031_114700-3cqkhntc/logs/debug-internal.log

@@ -0,0 +1,8 @@
+{"time":"2024-10-31T11:47:00.291622632-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-10-31T11:47:00.291649983-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_114700-3cqkhntc/logs/debug-core.log"}
+{"time":"2024-10-31T11:47:00.400110384-04:00","level":"INFO","msg":"created new stream","id":"3cqkhntc"}
+{"time":"2024-10-31T11:47:00.400155725-04:00","level":"INFO","msg":"stream: started","id":"3cqkhntc"}
+{"time":"2024-10-31T11:47:00.400207845-04:00","level":"INFO","msg":"sender: started","stream_id":"3cqkhntc"}
+{"time":"2024-10-31T11:47:00.400176745-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"3cqkhntc"}}
+{"time":"2024-10-31T11:47:00.400210445-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"3cqkhntc"}}
+{"time":"2024-10-31T11:47:00.639856673-04:00","level":"INFO","msg":"Starting system monitor"}

wandb/run-20241031_114700-3cqkhntc/logs/debug.log

@@ -0,0 +1,29 @@
+2024-10-31 11:47:00,285 INFO    MainThread:554143 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-10-31 11:47:00,286 INFO    MainThread:554143 [wandb_setup.py:_flush():79] Configure stats pid to 554143
+2024-10-31 11:47:00,286 INFO    MainThread:554143 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-10-31 11:47:00,286 INFO    MainThread:554143 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-10-31 11:47:00,286 INFO    MainThread:554143 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-31 11:47:00,286 INFO    MainThread:554143 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-31 11:47:00,286 INFO    MainThread:554143 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-10-31 11:47:00,286 INFO    MainThread:554143 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-31 11:47:00,286 INFO    MainThread:554143 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_114700-3cqkhntc/logs/debug.log
+2024-10-31 11:47:00,286 INFO    MainThread:554143 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_114700-3cqkhntc/logs/debug-internal.log
+2024-10-31 11:47:00,286 INFO    MainThread:554143 [wandb_init.py:init():621] calling init triggers
+2024-10-31 11:47:00,286 INFO    MainThread:554143 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-10-31 11:47:00,286 INFO    MainThread:554143 [wandb_init.py:init():671] starting backend
+2024-10-31 11:47:00,286 INFO    MainThread:554143 [wandb_init.py:init():675] sending inform_init request
+2024-10-31 11:47:00,288 INFO    MainThread:554143 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-31 11:47:00,288 INFO    MainThread:554143 [wandb_init.py:init():688] backend started and connected
+2024-10-31 11:47:00,293 INFO    MainThread:554143 [wandb_init.py:init():783] updated telemetry
+2024-10-31 11:47:00,326 INFO    MainThread:554143 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-10-31 11:47:00,636 INFO    MainThread:554143 [wandb_init.py:init():867] starting run threads in backend
+2024-10-31 11:47:00,724 INFO    MainThread:554143 [wandb_run.py:_console_start():2463] atexit reg
+2024-10-31 11:47:00,724 INFO    MainThread:554143 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-10-31 11:47:00,724 INFO    MainThread:554143 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-10-31 11:47:00,724 INFO    MainThread:554143 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-10-31 11:47:00,725 INFO    MainThread:554143 [wandb_init.py:init():911] run started, returning control to user process
+2024-10-31 11:47:00,725 INFO    MainThread:554143 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_full', 'train_set': '10M', 'batch_size': 3, 'epoch': 6, 'seed': 0, 'lr': 0.0001}
+2024-10-31 11:50:15,611 INFO    MainThread:554143 [wandb_run.py:_config_callback():1390] config_cb None None {'vocab_size': 128256, 'max_position_embeddings': 131072, 'hidden_size': 3072, 'intermediate_size': 8192, 'num_hidden_layers': 28, 'num_attention_heads': 24, 'num_key_value_heads': 8, 'hidden_act': 'silu', 'initializer_range': 0.02, 'rms_norm_eps': 1e-05, 'pretraining_tp': 1, 'use_cache': True, 'rope_theta': 500000.0, 'rope_scaling': {'factor': 32.0, 'high_freq_factor': 4.0, 'low_freq_factor': 1.0, 'original_max_position_embeddings': 8192, 'rope_type': 'llama3'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 128, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'bfloat16', 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': True, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['LlamaForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 128000, 'pad_token_id': None, 'eos_token_id': 128001, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'meta-llama/Llama-3.2-3B', 'transformers_version': '4.45.1', 'model_type': 'llama', 'output_dir': './checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': True, 'do_predict': False, 'eval_strategy': 'steps', 'prediction_loss_only': False, 'per_device_train_batch_size': 3, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 2, 'eval_accumulation_steps': None, 'eval_delay': 0, 'torch_empty_cache_steps': None, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 6, 'max_steps': -1, 'lr_scheduler_type': 'linear', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.0, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': './logs', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 1, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 150, 'save_total_limit': None, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 0, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': False, 'fp16': True, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 10, 'dataloader_num_workers': 0, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': './checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': 'deepspeed_config/train_dp_config.json', 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'evaluation_strategy': 'steps', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False, 'use_liger_kernel': False, 'eval_use_gather_object': False}
+2024-10-31 11:50:15,613 INFO    MainThread:554143 [wandb_config.py:__setitem__():154] config set model/num_parameters = 3212749824 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x7ff34dc14fd0>>
+2024-10-31 11:50:15,613 INFO    MainThread:554143 [wandb_run.py:_config_callback():1390] config_cb model/num_parameters 3212749824 None

wandb/run-20241031_114700-q0d78n2b/files/output.log

@@ -0,0 +1,43 @@
+Downloading shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [02:32<00:00, 76.42s/it]
+Loading checkpoint shards: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:04<00:00,  2.36s/it]
+tokenized_valid: Dataset({
+    features: ['input_ids', 'attention_mask'],
+    num_rows: 600
+})
+/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
+  warnings.warn(
+[2024-10-31 11:49:40,269] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2024-10-31 11:49:48,813] [INFO] [comm.py:652:init_distributed] cdb=None
+Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination
+Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...
+Loading extension module cpu_adam...
+Time to load cpu_adam op: 4.811358213424683 seconds
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 220, in <module>
+    trainer.train()
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/trainer.py", line 2052, in train
+    return inner_training_loop(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/trainer.py", line 2388, in _inner_training_loop
+    tr_loss_step = self.training_step(model, inputs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/trainer.py", line 3518, in training_step
+    self.accelerator.backward(loss, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/accelerate/accelerator.py", line 2238, in backward
+    self.deepspeed_engine_wrapped.backward(loss, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/accelerate/utils/deepspeed.py", line 186, in backward
+    self.engine.backward(loss, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/utils/nvtx.py", line 18, in wrapped_fn
+    ret_val = func(*args, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/runtime/engine.py", line 2020, in backward
+    self.optimizer.backward(loss, retain_graph=retain_graph)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/runtime/zero/stage_1_and_2.py", line 2063, in backward
+    self.loss_scaler.backward(loss.float(), retain_graph=retain_graph)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/runtime/fp16/loss_scaler.py", line 63, in backward
+    scaled_loss.backward(retain_graph=retain_graph)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/torch/_tensor.py", line 487, in backward
+    torch.autograd.backward(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/torch/autograd/__init__.py", line 200, in backward
+    Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
+KeyboardInterrupt
+Error in atexit._run_exitfuncs:
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/ops/transformer/inference/triton/matmul_ext.py", line 27, in is_nfs_path

wandb/run-20241101_012438-1onnw7d5/files/config.yaml

@@ -0,0 +1,49 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.9.19
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "3":
+                - 13
+                - 23
+                - 55
+            "4": 3.9.19
+            "5": 0.18.5
+            "6": 4.45.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+batch_size:
+    value: 3
+epoch:
+    value: 6
+lr:
+    value: 5e-06
+perturbation:
+    value: shuffle_nodeterministic
+seed:
+    value: 0
+train_set:
+    value: 10M

wandb/run-20241101_012438-1onnw7d5/files/output.log

@@ -0,0 +1,12 @@
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 164, in <module>
+    dataset = load_dataset('babylm_dataset_test.py', name=dataset_name, trust_remote_code=True)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/load.py", line 2074, in load_dataset
+    builder_instance = load_dataset_builder(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/load.py", line 1832, in load_dataset_builder
+    builder_instance: DatasetBuilder = builder_cls(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/builder.py", line 342, in __init__
+    self.config, self.config_id = self._create_builder_config(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/builder.py", line 569, in _create_builder_config
+    raise ValueError(
+ValueError: BuilderConfig 'babylm_shuffle_nodeterministic_10M_seed0' not found. Available: ['babylm_hop_control_10M_seed0', 'babylm_hop_tokens4_10M_seed0', 'babylm_hop_words4_10M_seed0', 'babylm_reverse_control_10M_seed0', 'babylm_reverse_partial_10M_seed0', 'babylm_reverse_full_10M_seed0', 'babylm_shuffle_control_10M_seed0', 'babylm_shuffle_nondeterministic_10M_seed0', 'babylm_shuffle_deterministic21_10M_seed0', 'babylm_shuffle_deterministic57_10M_seed0', 'babylm_shuffle_deterministic84_10M_seed0', 'babylm_shuffle_local3_10M_seed0', 'babylm_shuffle_local5_10M_seed0', 'babylm_shuffle_local10_10M_seed0', 'babylm_shuffle_even_odd_10M_seed0']

wandb/run-20241101_012438-1onnw7d5/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-11-01T05:24:38.172065Z",
+  "args":  [
+    "--perturbation",
+    "shuffle_nodeterministic",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "6",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1753992159232"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241101_012438-1onnw7d5/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":0}}

wandb/run-20241101_012438-1onnw7d5/logs/debug-internal.log

@@ -0,0 +1,16 @@
+{"time":"2024-11-01T01:24:38.174471309-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-11-01T01:24:38.174485399-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_012438-1onnw7d5/logs/debug-core.log"}
+{"time":"2024-11-01T01:24:38.280498427-04:00","level":"INFO","msg":"created new stream","id":"1onnw7d5"}
+{"time":"2024-11-01T01:24:38.280533768-04:00","level":"INFO","msg":"stream: started","id":"1onnw7d5"}
+{"time":"2024-11-01T01:24:38.280568298-04:00","level":"INFO","msg":"sender: started","stream_id":"1onnw7d5"}
+{"time":"2024-11-01T01:24:38.280549488-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"1onnw7d5"}}
+{"time":"2024-11-01T01:24:38.280570018-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"1onnw7d5"}}
+{"time":"2024-11-01T01:24:38.441821679-04:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-11-01T01:24:38.744246503-04:00","level":"INFO","msg":"stream: closing","id":"1onnw7d5"}
+{"time":"2024-11-01T01:24:38.744301793-04:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-11-01T01:24:38.744928418-04:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-11-01T01:24:39.133873344-04:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-11-01T01:24:39.266150554-04:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"1onnw7d5"}}
+{"time":"2024-11-01T01:24:39.266210184-04:00","level":"INFO","msg":"sender: closed","stream_id":"1onnw7d5"}
+{"time":"2024-11-01T01:24:39.266202684-04:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"1onnw7d5"}}
+{"time":"2024-11-01T01:24:39.266298455-04:00","level":"INFO","msg":"stream: closed","id":"1onnw7d5"}

wandb/run-20241101_012438-1onnw7d5/logs/debug.log

@@ -0,0 +1,27 @@
+2024-11-01 01:24:38,170 INFO    MainThread:676349 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-11-01 01:24:38,170 INFO    MainThread:676349 [wandb_setup.py:_flush():79] Configure stats pid to 676349
+2024-11-01 01:24:38,170 INFO    MainThread:676349 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-11-01 01:24:38,170 INFO    MainThread:676349 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-11-01 01:24:38,170 INFO    MainThread:676349 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-11-01 01:24:38,170 INFO    MainThread:676349 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-11-01 01:24:38,170 INFO    MainThread:676349 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-11-01 01:24:38,170 INFO    MainThread:676349 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-11-01 01:24:38,170 INFO    MainThread:676349 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_012438-1onnw7d5/logs/debug.log
+2024-11-01 01:24:38,170 INFO    MainThread:676349 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_012438-1onnw7d5/logs/debug-internal.log
+2024-11-01 01:24:38,170 INFO    MainThread:676349 [wandb_init.py:init():621] calling init triggers
+2024-11-01 01:24:38,170 INFO    MainThread:676349 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-11-01 01:24:38,170 INFO    MainThread:676349 [wandb_init.py:init():671] starting backend
+2024-11-01 01:24:38,170 INFO    MainThread:676349 [wandb_init.py:init():675] sending inform_init request
+2024-11-01 01:24:38,171 INFO    MainThread:676349 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-11-01 01:24:38,171 INFO    MainThread:676349 [wandb_init.py:init():688] backend started and connected
+2024-11-01 01:24:38,176 INFO    MainThread:676349 [wandb_init.py:init():783] updated telemetry
+2024-11-01 01:24:38,210 INFO    MainThread:676349 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-11-01 01:24:38,438 INFO    MainThread:676349 [wandb_init.py:init():867] starting run threads in backend
+2024-11-01 01:24:38,576 INFO    MainThread:676349 [wandb_run.py:_console_start():2463] atexit reg
+2024-11-01 01:24:38,576 INFO    MainThread:676349 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-11-01 01:24:38,576 INFO    MainThread:676349 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-11-01 01:24:38,576 INFO    MainThread:676349 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-11-01 01:24:38,579 INFO    MainThread:676349 [wandb_init.py:init():911] run started, returning control to user process
+2024-11-01 01:24:38,579 INFO    MainThread:676349 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_nodeterministic', 'train_set': '10M', 'batch_size': 3, 'epoch': 6, 'seed': 0, 'lr': 5e-06}
+2024-11-01 01:24:38,744 WARNING MsgRouterThr:676349 [router.py:message_loop():77] message_loop has been closed

wandb/run-20241101_012612-q08jbqqf/files/output.log

@@ -0,0 +1,12 @@
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 164, in <module>
+    dataset = load_dataset('babylm_dataset_test.py', name=dataset_name, trust_remote_code=True)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/load.py", line 2074, in load_dataset
+    builder_instance = load_dataset_builder(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/load.py", line 1832, in load_dataset_builder
+    builder_instance: DatasetBuilder = builder_cls(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/builder.py", line 342, in __init__
+    self.config, self.config_id = self._create_builder_config(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/builder.py", line 569, in _create_builder_config
+    raise ValueError(
+ValueError: BuilderConfig 'babylm_shuffle_nodeterministic_10M_seed0' not found. Available: ['babylm_hop_control_10M_seed0', 'babylm_hop_tokens4_10M_seed0', 'babylm_hop_words4_10M_seed0', 'babylm_reverse_control_10M_seed0', 'babylm_reverse_partial_10M_seed0', 'babylm_reverse_full_10M_seed0', 'babylm_shuffle_control_10M_seed0', 'babylm_shuffle_nondeterministic_10M_seed0', 'babylm_shuffle_deterministic21_10M_seed0', 'babylm_shuffle_deterministic57_10M_seed0', 'babylm_shuffle_deterministic84_10M_seed0', 'babylm_shuffle_local3_10M_seed0', 'babylm_shuffle_local5_10M_seed0', 'babylm_shuffle_local10_10M_seed0', 'babylm_shuffle_even_odd_10M_seed0']

wandb/run-20241101_012613-k6o0lha8/files/config.yaml

@@ -0,0 +1,49 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.9.19
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "3":
+                - 13
+                - 23
+                - 55
+            "4": 3.9.19
+            "5": 0.18.5
+            "6": 4.45.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+batch_size:
+    value: 3
+epoch:
+    value: 6
+lr:
+    value: 5e-06
+perturbation:
+    value: shuffle_nodeterministic
+seed:
+    value: 0
+train_set:
+    value: 10M

wandb/run-20241101_012613-k6o0lha8/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":0}}

wandb/run-20241101_012613-k6o0lha8/logs/debug-internal.log

@@ -0,0 +1,16 @@
+{"time":"2024-11-01T01:26:13.053618839-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-11-01T01:26:13.053632699-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_012613-k6o0lha8/logs/debug-core.log"}
+{"time":"2024-11-01T01:26:13.161223586-04:00","level":"INFO","msg":"created new stream","id":"k6o0lha8"}
+{"time":"2024-11-01T01:26:13.161267266-04:00","level":"INFO","msg":"stream: started","id":"k6o0lha8"}
+{"time":"2024-11-01T01:26:13.161303247-04:00","level":"INFO","msg":"sender: started","stream_id":"k6o0lha8"}
+{"time":"2024-11-01T01:26:13.161289557-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"k6o0lha8"}}
+{"time":"2024-11-01T01:26:13.161370687-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"k6o0lha8"}}
+{"time":"2024-11-01T01:26:13.366986845-04:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-11-01T01:26:13.481739202-04:00","level":"INFO","msg":"stream: closing","id":"k6o0lha8"}
+{"time":"2024-11-01T01:26:13.481812843-04:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-11-01T01:26:13.482462557-04:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-11-01T01:26:14.147796598-04:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-11-01T01:26:14.269127141-04:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"k6o0lha8"}}
+{"time":"2024-11-01T01:26:14.269155371-04:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"k6o0lha8"}}
+{"time":"2024-11-01T01:26:14.269185711-04:00","level":"INFO","msg":"sender: closed","stream_id":"k6o0lha8"}
+{"time":"2024-11-01T01:26:14.269277432-04:00","level":"INFO","msg":"stream: closed","id":"k6o0lha8"}

wandb/run-20241101_012613-k6o0lha8/logs/debug.log

@@ -0,0 +1,27 @@
+2024-11-01 01:26:13,049 INFO    MainThread:677635 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-11-01 01:26:13,049 INFO    MainThread:677635 [wandb_setup.py:_flush():79] Configure stats pid to 677635
+2024-11-01 01:26:13,049 INFO    MainThread:677635 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-11-01 01:26:13,049 INFO    MainThread:677635 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-11-01 01:26:13,049 INFO    MainThread:677635 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-11-01 01:26:13,049 INFO    MainThread:677635 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-11-01 01:26:13,049 INFO    MainThread:677635 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-11-01 01:26:13,049 INFO    MainThread:677635 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-11-01 01:26:13,049 INFO    MainThread:677635 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_012613-k6o0lha8/logs/debug.log
+2024-11-01 01:26:13,049 INFO    MainThread:677635 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_012613-k6o0lha8/logs/debug-internal.log
+2024-11-01 01:26:13,049 INFO    MainThread:677635 [wandb_init.py:init():621] calling init triggers
+2024-11-01 01:26:13,049 INFO    MainThread:677635 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-11-01 01:26:13,049 INFO    MainThread:677635 [wandb_init.py:init():671] starting backend
+2024-11-01 01:26:13,049 INFO    MainThread:677635 [wandb_init.py:init():675] sending inform_init request
+2024-11-01 01:26:13,050 INFO    MainThread:677635 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-11-01 01:26:13,051 INFO    MainThread:677635 [wandb_init.py:init():688] backend started and connected
+2024-11-01 01:26:13,054 INFO    MainThread:677635 [wandb_init.py:init():783] updated telemetry
+2024-11-01 01:26:13,084 INFO    MainThread:677635 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-11-01 01:26:13,364 INFO    MainThread:677635 [wandb_init.py:init():867] starting run threads in backend
+2024-11-01 01:26:13,467 INFO    MainThread:677635 [wandb_run.py:_console_start():2463] atexit reg
+2024-11-01 01:26:13,467 INFO    MainThread:677635 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-11-01 01:26:13,467 INFO    MainThread:677635 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-11-01 01:26:13,467 INFO    MainThread:677635 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-11-01 01:26:13,469 INFO    MainThread:677635 [wandb_init.py:init():911] run started, returning control to user process
+2024-11-01 01:26:13,469 INFO    MainThread:677635 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_nodeterministic', 'train_set': '10M', 'batch_size': 3, 'epoch': 6, 'seed': 0, 'lr': 5e-06}
+2024-11-01 01:26:13,481 WARNING MsgRouterThr:677635 [router.py:message_loop():77] message_loop has been closed

wandb/run-20241101_092804-qhsuxbxe/files/output.log

@@ -0,0 +1,14 @@
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:14<00:00,  7.48s/it]
+Map: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 17013/17013 [00:49<00:00, 346.42 examples/s]
+tokenized_valid: Dataset({
+    features: ['input_ids', 'attention_mask'],
+    num_rows: 600
+})
+/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
+  warnings.warn(
+[2024-11-01 09:29:10,914] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2024-11-01 09:29:22,407] [INFO] [comm.py:652:init_distributed] cdb=None
+Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination
+Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...
+Loading extension module cpu_adam...
+Time to load cpu_adam op: 4.957691431045532 seconds

wandb/run-20241101_092804-qhsuxbxe/files/requirements.txt

@@ -0,0 +1,147 @@
+funcsigs==1.0.2
+sentry-sdk==2.17.0
+multiprocess==0.70.16
+numpy==1.26.2
+pluralizer==1.2.0
+debugpy==1.6.7
+nvidia-cudnn-cu11==8.5.0.96
+deepspeed==0.15.2
+data==0.4
+pandas==2.1.3
+tomli==2.0.1
+charset-normalizer==3.3.2
+attrs==24.2.0
+aiosignal==1.3.1
+fsspec==2023.10.0
+nvidia-cusparse-cu11==11.7.4.91
+zipp==3.12.0
+mypy-extensions==1.0.0
+datasets==3.0.1
+joblib==1.3.2
+hjson==3.1.0
+traitlets==5.7.1
+stack-data==0.6.0
+transformers==4.45.1
+sympy==1.11.1
+Pygments==2.15.0
+docker-pycreds==0.4.0
+dill==0.3.8
+wheel==0.44.0
+prompt-toolkit==3.0.30
+parso==0.8.3
+ipykernel==6.23.1
+pyarrow==17.0.0
+certifi==2023.11.17
+nvidia-cufft-cu11==10.9.0.58
+six==1.16.0
+pydantic==2.9.2
+click==8.1.7
+nest-asyncio==1.5.6
+gmpy2==2.1.0
+matplotlib==3.8.2
+scipy==1.11.4
+typing_extensions==4.12.2
+statsmodels==0.14.0
+huggingface-hub==0.25.0
+frozenlist==1.4.1
+gpustat==1.1.1
+nvidia-nvtx-cu11==11.7.91
+safetensors==0.4.5
+stanza==1.9.2
+decorator==5.1.1
+seaborn==0.13.0
+sentencepiece==0.2.0
+PyYAML==6.0.1
+black==24.8.0
+protobuf==4.25.1
+pickleshare==0.7.5
+peft==0.13.0
+triton==2.0.0
+nvidia-cuda-runtime-cu11==11.7.99
+Jinja2==3.1.2
+nvidia-cusolver-cu11==11.4.0.1
+executing==1.2.0
+jupyter_client==8.1.0
+pluggy==1.3.0
+cmake==3.30.3
+pytz==2023.3.post1
+aiohappyeyeballs==2.4.2
+kiwisolver==1.4.5
+py-cpuinfo==9.0.0
+Pillow==10.1.0
+ptyprocess==0.7.0
+importlib_resources==6.4.5
+GitPython==3.1.43
+importlib-metadata==6.0.0
+iniconfig==2.0.0
+scikit-learn==1.3.2
+exceptiongroup==1.1.0
+networkx==2.8.6
+accelerate==1.0.0
+nltk==3.8.1
+shutilwhich==1.1.0
+fonttools==4.45.1
+future==0.18.3
+aiohttp==3.10.6
+wcwidth==0.2.5
+idna==3.6
+filelock==3.12.2
+pathspec==0.12.1
+jupyter_core==5.1.0
+lit==18.1.8
+nvidia-curand-cu11==10.2.10.91
+nvidia-cublas-cu11==11.10.3.66
+nvidia-ml-py==12.560.30
+msgpack==1.1.0
+python-dateutil==2.8.2
+blessed==1.20.0
+packaging==23.0
+gitdb==4.0.11
+yarl==1.13.0
+emoji==2.8.0
+tzdata==2023.3
+cycler==0.12.1
+tornado==6.2
+backcall==0.2.0
+plotnine==0.12.4
+ninja==1.11.1.1
+latex==0.7.0
+wandb==0.18.5
+setproctitle==1.3.3
+threadpoolctl==3.2.0
+requests==2.32.3
+pyparsing==3.1.1
+smmap==5.0.1
+pyzmq==23.0.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+matplotlib-inline==0.1.6
+latexcodec==1.0.0
+ipython==8.0.0
+patsy==0.5.3
+contourpy==1.2.0
+multidict==6.1.0
+mizani==0.9.3
+urllib3==2.1.0
+tokenizers==0.20.0
+MarkupSafe==2.1.2
+pip==24.2
+pexpect==4.8.0
+tqdm==4.66.5
+jedi==0.18.2
+pydantic_core==2.23.4
+tempdir==0.7.1
+mpmath==1.2.1
+setuptools==72.1.0
+pytest==7.4.3
+pure-eval==0.2.2
+psutil==5.9.1
+comm==0.1.2
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+regex==2023.10.3
+platformdirs==2.5.2
+asttokens==2.2.1
+torch==2.0.0
+nvidia-nccl-cu11==2.14.3
+xxhash==3.5.0

wandb/run-20241101_092804-qhsuxbxe/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-11-01T13:28:04.179365Z",
+  "args":  [
+    "--perturbation",
+    "shuffle_nondeterministic",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "6",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1754628071424"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241101_092804-qhsuxbxe/logs/debug-internal.log

@@ -0,0 +1,8 @@
+{"time":"2024-11-01T09:28:04.182358567-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-11-01T09:28:04.182375607-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_092804-qhsuxbxe/logs/debug-core.log"}
+{"time":"2024-11-01T09:28:04.291337625-04:00","level":"INFO","msg":"created new stream","id":"qhsuxbxe"}
+{"time":"2024-11-01T09:28:04.291389815-04:00","level":"INFO","msg":"stream: started","id":"qhsuxbxe"}
+{"time":"2024-11-01T09:28:04.291442536-04:00","level":"INFO","msg":"sender: started","stream_id":"qhsuxbxe"}
+{"time":"2024-11-01T09:28:04.291424526-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"qhsuxbxe"}}
+{"time":"2024-11-01T09:28:04.291463006-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"qhsuxbxe"}}
+{"time":"2024-11-01T09:28:04.534676015-04:00","level":"INFO","msg":"Starting system monitor"}

wandb/run-20241101_092804-qhsuxbxe/logs/debug.log

@@ -0,0 +1,26 @@
+2024-11-01 09:28:04,177 INFO    MainThread:779816 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-11-01 09:28:04,177 INFO    MainThread:779816 [wandb_setup.py:_flush():79] Configure stats pid to 779816
+2024-11-01 09:28:04,177 INFO    MainThread:779816 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-11-01 09:28:04,177 INFO    MainThread:779816 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-11-01 09:28:04,177 INFO    MainThread:779816 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-11-01 09:28:04,177 INFO    MainThread:779816 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-11-01 09:28:04,177 INFO    MainThread:779816 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-11-01 09:28:04,177 INFO    MainThread:779816 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-11-01 09:28:04,177 INFO    MainThread:779816 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_092804-qhsuxbxe/logs/debug.log
+2024-11-01 09:28:04,177 INFO    MainThread:779816 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_092804-qhsuxbxe/logs/debug-internal.log
+2024-11-01 09:28:04,177 INFO    MainThread:779816 [wandb_init.py:init():621] calling init triggers
+2024-11-01 09:28:04,177 INFO    MainThread:779816 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-11-01 09:28:04,177 INFO    MainThread:779816 [wandb_init.py:init():671] starting backend
+2024-11-01 09:28:04,177 INFO    MainThread:779816 [wandb_init.py:init():675] sending inform_init request
+2024-11-01 09:28:04,178 INFO    MainThread:779816 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-11-01 09:28:04,179 INFO    MainThread:779816 [wandb_init.py:init():688] backend started and connected
+2024-11-01 09:28:04,182 INFO    MainThread:779816 [wandb_init.py:init():783] updated telemetry
+2024-11-01 09:28:04,218 INFO    MainThread:779816 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-11-01 09:28:04,531 INFO    MainThread:779816 [wandb_init.py:init():867] starting run threads in backend
+2024-11-01 09:28:04,647 INFO    MainThread:779816 [wandb_run.py:_console_start():2463] atexit reg
+2024-11-01 09:28:04,647 INFO    MainThread:779816 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-11-01 09:28:04,647 INFO    MainThread:779816 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-11-01 09:28:04,647 INFO    MainThread:779816 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-11-01 09:28:04,649 INFO    MainThread:779816 [wandb_init.py:init():911] run started, returning control to user process
+2024-11-01 09:28:04,649 INFO    MainThread:779816 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_nondeterministic', 'train_set': '10M', 'batch_size': 3, 'epoch': 6, 'seed': 0, 'lr': 5e-06}

wandb/run-20241101_201707-qparlnlu/files/output.log

@@ -0,0 +1,16 @@
+Loading checkpoint shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:04<00:00,  2.21s/it]
+tokenized_valid: Dataset({
+    features: ['input_ids', 'attention_mask'],
+    num_rows: 600
+})
+/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
+  warnings.warn(
+[2024-11-01 20:17:14,306] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2024-11-01 20:17:22,315] [INFO] [comm.py:652:init_distributed] cdb=None
+Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination
+Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...
+Emitting ninja build file /home/chunhui/.cache/torch_extensions/py39_cu117/cpu_adam/build.ninja...
+Building extension module cpu_adam...
+Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N)
+Loading extension module cpu_adam...
+Time to load cpu_adam op: 5.1879143714904785 seconds

wandb/run-20241101_201707-qparlnlu/files/requirements.txt

@@ -0,0 +1,147 @@
+funcsigs==1.0.2
+sentry-sdk==2.17.0
+multiprocess==0.70.16
+numpy==1.26.2
+pluralizer==1.2.0
+debugpy==1.6.7
+nvidia-cudnn-cu11==8.5.0.96
+deepspeed==0.15.2
+data==0.4
+pandas==2.1.3
+tomli==2.0.1
+charset-normalizer==3.3.2
+attrs==24.2.0
+aiosignal==1.3.1
+fsspec==2023.10.0
+nvidia-cusparse-cu11==11.7.4.91
+zipp==3.12.0
+mypy-extensions==1.0.0
+datasets==3.0.1
+joblib==1.3.2
+hjson==3.1.0
+traitlets==5.7.1
+stack-data==0.6.0
+transformers==4.45.1
+sympy==1.11.1
+Pygments==2.15.0
+docker-pycreds==0.4.0
+dill==0.3.8
+wheel==0.44.0
+prompt-toolkit==3.0.30
+parso==0.8.3
+ipykernel==6.23.1
+pyarrow==17.0.0
+certifi==2023.11.17
+nvidia-cufft-cu11==10.9.0.58
+six==1.16.0
+pydantic==2.9.2
+click==8.1.7
+nest-asyncio==1.5.6
+gmpy2==2.1.0
+matplotlib==3.8.2
+scipy==1.11.4
+typing_extensions==4.12.2
+statsmodels==0.14.0
+huggingface-hub==0.25.0
+frozenlist==1.4.1
+gpustat==1.1.1
+nvidia-nvtx-cu11==11.7.91
+safetensors==0.4.5
+stanza==1.9.2
+decorator==5.1.1
+seaborn==0.13.0
+sentencepiece==0.2.0
+PyYAML==6.0.1
+black==24.8.0
+protobuf==4.25.1
+pickleshare==0.7.5
+peft==0.13.0
+triton==2.0.0
+nvidia-cuda-runtime-cu11==11.7.99
+Jinja2==3.1.2
+nvidia-cusolver-cu11==11.4.0.1
+executing==1.2.0
+jupyter_client==8.1.0
+pluggy==1.3.0
+cmake==3.30.3
+pytz==2023.3.post1
+aiohappyeyeballs==2.4.2
+kiwisolver==1.4.5
+py-cpuinfo==9.0.0
+Pillow==10.1.0
+ptyprocess==0.7.0
+importlib_resources==6.4.5
+GitPython==3.1.43
+importlib-metadata==6.0.0
+iniconfig==2.0.0
+scikit-learn==1.3.2
+exceptiongroup==1.1.0
+networkx==2.8.6
+accelerate==1.0.0
+nltk==3.8.1
+shutilwhich==1.1.0
+fonttools==4.45.1
+future==0.18.3
+aiohttp==3.10.6
+wcwidth==0.2.5
+idna==3.6
+filelock==3.12.2
+pathspec==0.12.1
+jupyter_core==5.1.0
+lit==18.1.8
+nvidia-curand-cu11==10.2.10.91
+nvidia-cublas-cu11==11.10.3.66
+nvidia-ml-py==12.560.30
+msgpack==1.1.0
+python-dateutil==2.8.2
+blessed==1.20.0
+packaging==23.0
+gitdb==4.0.11
+yarl==1.13.0
+emoji==2.8.0
+tzdata==2023.3
+cycler==0.12.1
+tornado==6.2
+backcall==0.2.0
+plotnine==0.12.4
+ninja==1.11.1.1
+latex==0.7.0
+wandb==0.18.5
+setproctitle==1.3.3
+threadpoolctl==3.2.0
+requests==2.32.3
+pyparsing==3.1.1
+smmap==5.0.1
+pyzmq==23.0.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+matplotlib-inline==0.1.6
+latexcodec==1.0.0
+ipython==8.0.0
+patsy==0.5.3
+contourpy==1.2.0
+multidict==6.1.0
+mizani==0.9.3
+urllib3==2.1.0
+tokenizers==0.20.0
+MarkupSafe==2.1.2
+pip==24.2
+pexpect==4.8.0
+tqdm==4.66.5
+jedi==0.18.2
+pydantic_core==2.23.4
+tempdir==0.7.1
+mpmath==1.2.1
+setuptools==72.1.0
+pytest==7.4.3
+pure-eval==0.2.2
+psutil==5.9.1
+comm==0.1.2
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+regex==2023.10.3
+platformdirs==2.5.2
+asttokens==2.2.1
+torch==2.0.0
+nvidia-nccl-cu11==2.14.3
+xxhash==3.5.0

wandb/run-20241101_201707-qparlnlu/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-11-02T00:17:07.873886Z",
+  "args":  [
+    "--perturbation",
+    "shuffle_nondeterministic",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "3",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1754802667520"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241101_201707-qparlnlu/logs/debug-internal.log

@@ -0,0 +1,8 @@
+{"time":"2024-11-01T20:17:07.876661786-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-11-01T20:17:07.876685067-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_201707-qparlnlu/logs/debug-core.log"}
+{"time":"2024-11-01T20:17:07.984430937-04:00","level":"INFO","msg":"created new stream","id":"qparlnlu"}
+{"time":"2024-11-01T20:17:07.984473777-04:00","level":"INFO","msg":"stream: started","id":"qparlnlu"}
+{"time":"2024-11-01T20:17:07.984539197-04:00","level":"INFO","msg":"sender: started","stream_id":"qparlnlu"}
+{"time":"2024-11-01T20:17:07.984500437-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"qparlnlu"}}
+{"time":"2024-11-01T20:17:07.984517717-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"qparlnlu"}}
+{"time":"2024-11-01T20:17:08.155922608-04:00","level":"INFO","msg":"Starting system monitor"}

wandb/run-20241101_201707-qparlnlu/logs/debug.log

@@ -0,0 +1,26 @@
+2024-11-01 20:17:07,870 INFO    MainThread:875623 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-11-01 20:17:07,870 INFO    MainThread:875623 [wandb_setup.py:_flush():79] Configure stats pid to 875623
+2024-11-01 20:17:07,870 INFO    MainThread:875623 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-11-01 20:17:07,871 INFO    MainThread:875623 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-11-01 20:17:07,871 INFO    MainThread:875623 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-11-01 20:17:07,871 INFO    MainThread:875623 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-11-01 20:17:07,871 INFO    MainThread:875623 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-11-01 20:17:07,871 INFO    MainThread:875623 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-11-01 20:17:07,871 INFO    MainThread:875623 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_201707-qparlnlu/logs/debug.log
+2024-11-01 20:17:07,871 INFO    MainThread:875623 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_201707-qparlnlu/logs/debug-internal.log
+2024-11-01 20:17:07,871 INFO    MainThread:875623 [wandb_init.py:init():621] calling init triggers
+2024-11-01 20:17:07,871 INFO    MainThread:875623 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-11-01 20:17:07,871 INFO    MainThread:875623 [wandb_init.py:init():671] starting backend
+2024-11-01 20:17:07,871 INFO    MainThread:875623 [wandb_init.py:init():675] sending inform_init request
+2024-11-01 20:17:07,873 INFO    MainThread:875623 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-11-01 20:17:07,873 INFO    MainThread:875623 [wandb_init.py:init():688] backend started and connected
+2024-11-01 20:17:07,877 INFO    MainThread:875623 [wandb_init.py:init():783] updated telemetry
+2024-11-01 20:17:07,900 INFO    MainThread:875623 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-11-01 20:17:08,152 INFO    MainThread:875623 [wandb_init.py:init():867] starting run threads in backend
+2024-11-01 20:17:08,239 INFO    MainThread:875623 [wandb_run.py:_console_start():2463] atexit reg
+2024-11-01 20:17:08,239 INFO    MainThread:875623 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-11-01 20:17:08,239 INFO    MainThread:875623 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-11-01 20:17:08,239 INFO    MainThread:875623 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-11-01 20:17:08,241 INFO    MainThread:875623 [wandb_init.py:init():911] run started, returning control to user process
+2024-11-01 20:17:08,241 INFO    MainThread:875623 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_nondeterministic', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0, 'lr': 5e-06}

wandb/run-20241105_161832-sl992h9i/files/output.log

@@ -0,0 +1,28 @@
+Downloading shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [02:33<00:00, 76.52s/it]
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:09<00:00,  4.70s/it]
+Map: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 16425/16425 [00:45<00:00, 360.17 examples/s]
+Map: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 17013/17013 [00:45<00:00, 371.27 examples/s]
+tokenized_valid: Dataset({
+    features: ['input_ids', 'attention_mask'],
+    num_rows: 1000
+})
+/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
+  warnings.warn(
+[2024-11-05 16:23:11,215] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2024-11-05 16:23:22,612] [INFO] [comm.py:652:init_distributed] cdb=None
+[2024-11-05 16:23:22,613] [INFO] [comm.py:683:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl
+Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.
+Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination
+Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...
+Loading extension module cpu_adam...
+Time to load cpu_adam op: 31.5544273853302 seconds
+wandb: WARNING The `run_name` is currently set to the same value as `TrainingArguments.output_dir`. If this was not intended, please specify a different run name by setting the `TrainingArguments.run_name` parameter.
+  0%|▍                                                                                                                                              | 8/2736 [00:37<3:24:38,  4.50s/it]
+{'loss': 3.0476, 'grad_norm': 0.0, 'learning_rate': 0.0, 'epoch': 0.0}
+{'loss': 3.0289, 'grad_norm': 0.0, 'learning_rate': 0.0, 'epoch': 0.0}
+{'loss': 2.9935, 'grad_norm': 0.0, 'learning_rate': 0.0, 'epoch': 0.0}
+{'loss': 3.0139, 'grad_norm': 0.0, 'learning_rate': 0.0, 'epoch': 0.0}
+{'loss': 2.9892, 'grad_norm': 0.0, 'learning_rate': 0.0, 'epoch': 0.01}
+{'loss': 3.1252, 'grad_norm': 0.0, 'learning_rate': 0.0, 'epoch': 0.01}
+{'loss': 3.0291, 'grad_norm': 0.0, 'learning_rate': 0.0, 'epoch': 0.01}
+{'loss': 3.0607, 'grad_norm': 0.0, 'learning_rate': 0.0, 'epoch': 0.01}

wandb/run-20241105_161832-sl992h9i/files/requirements.txt

@@ -0,0 +1,147 @@
+funcsigs==1.0.2
+sentry-sdk==2.17.0
+multiprocess==0.70.16
+numpy==1.26.2
+pluralizer==1.2.0
+debugpy==1.6.7
+nvidia-cudnn-cu11==8.5.0.96
+deepspeed==0.15.2
+data==0.4
+pandas==2.1.3
+tomli==2.0.1
+charset-normalizer==3.3.2
+attrs==24.2.0
+aiosignal==1.3.1
+fsspec==2023.10.0
+nvidia-cusparse-cu11==11.7.4.91
+zipp==3.12.0
+mypy-extensions==1.0.0
+datasets==3.0.1
+joblib==1.3.2
+hjson==3.1.0
+traitlets==5.7.1
+stack-data==0.6.0
+transformers==4.45.1
+sympy==1.11.1
+Pygments==2.15.0
+docker-pycreds==0.4.0
+dill==0.3.8
+wheel==0.44.0
+prompt-toolkit==3.0.30
+parso==0.8.3
+ipykernel==6.23.1
+pyarrow==17.0.0
+certifi==2023.11.17
+nvidia-cufft-cu11==10.9.0.58
+six==1.16.0
+pydantic==2.9.2
+click==8.1.7
+nest-asyncio==1.5.6
+gmpy2==2.1.0
+matplotlib==3.8.2
+scipy==1.11.4
+typing_extensions==4.12.2
+statsmodels==0.14.0
+huggingface-hub==0.25.0
+frozenlist==1.4.1
+gpustat==1.1.1
+nvidia-nvtx-cu11==11.7.91
+safetensors==0.4.5
+stanza==1.9.2
+decorator==5.1.1
+seaborn==0.13.0
+sentencepiece==0.2.0
+PyYAML==6.0.1
+black==24.8.0
+protobuf==4.25.1
+pickleshare==0.7.5
+peft==0.13.0
+triton==2.0.0
+nvidia-cuda-runtime-cu11==11.7.99
+Jinja2==3.1.2
+nvidia-cusolver-cu11==11.4.0.1
+executing==1.2.0
+jupyter_client==8.1.0
+pluggy==1.3.0
+cmake==3.30.3
+pytz==2023.3.post1
+aiohappyeyeballs==2.4.2
+kiwisolver==1.4.5
+py-cpuinfo==9.0.0
+Pillow==10.1.0
+ptyprocess==0.7.0
+importlib_resources==6.4.5
+GitPython==3.1.43
+importlib-metadata==6.0.0
+iniconfig==2.0.0
+scikit-learn==1.3.2
+exceptiongroup==1.1.0
+networkx==2.8.6
+accelerate==1.0.0
+nltk==3.8.1
+shutilwhich==1.1.0
+fonttools==4.45.1
+future==0.18.3
+aiohttp==3.10.6
+wcwidth==0.2.5
+idna==3.6
+filelock==3.12.2
+pathspec==0.12.1
+jupyter_core==5.1.0
+lit==18.1.8
+nvidia-curand-cu11==10.2.10.91
+nvidia-cublas-cu11==11.10.3.66
+nvidia-ml-py==12.560.30
+msgpack==1.1.0
+python-dateutil==2.8.2
+blessed==1.20.0
+packaging==23.0
+gitdb==4.0.11
+yarl==1.13.0
+emoji==2.8.0
+tzdata==2023.3
+cycler==0.12.1
+tornado==6.2
+backcall==0.2.0
+plotnine==0.12.4
+ninja==1.11.1.1
+latex==0.7.0
+wandb==0.18.5
+setproctitle==1.3.3
+threadpoolctl==3.2.0
+requests==2.32.3
+pyparsing==3.1.1
+smmap==5.0.1
+pyzmq==23.0.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+matplotlib-inline==0.1.6
+latexcodec==1.0.0
+ipython==8.0.0
+patsy==0.5.3
+contourpy==1.2.0
+multidict==6.1.0
+mizani==0.9.3
+urllib3==2.1.0
+tokenizers==0.20.0
+MarkupSafe==2.1.2
+pip==24.2
+pexpect==4.8.0
+tqdm==4.66.5
+jedi==0.18.2
+pydantic_core==2.23.4
+tempdir==0.7.1
+mpmath==1.2.1
+setuptools==72.1.0
+pytest==7.4.3
+pure-eval==0.2.2
+psutil==5.9.1
+comm==0.1.2
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+regex==2023.10.3
+platformdirs==2.5.2
+asttokens==2.2.1
+torch==2.0.0
+nvidia-nccl-cu11==2.14.3
+xxhash==3.5.0

wandb/run-20241105_161832-sl992h9i/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-11-05T21:18:32.539569Z",
+  "args":  [
+    "--perturbation",
+    "shuffle_deterministic21",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "3",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1785200173056"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241105_162824-9m8ga5sn/files/output.log

@@ -0,0 +1,39 @@
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:05<00:00,  2.69s/it]
+Map: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 17013/17013 [00:47<00:00, 358.89 examples/s]
+tokenized_valid: Dataset({
+    features: ['input_ids', 'attention_mask'],
+    num_rows: 1000
+})
+/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
+  warnings.warn(
+[2024-11-05 16:29:19,412] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2024-11-05 16:29:28,405] [INFO] [comm.py:652:init_distributed] cdb=None
+Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination
+Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...
+Loading extension module cpu_adam...
+Time to load cpu_adam op: 5.117670059204102 seconds
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 222, in <module>
+    trainer.train()
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/trainer.py", line 2052, in train
+    return inner_training_loop(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/trainer.py", line 2204, in _inner_training_loop
+    model, self.optimizer = self.accelerator.prepare(self.model, self.optimizer)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/accelerate/accelerator.py", line 1344, in prepare
+    result = self._prepare_deepspeed(*args)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/accelerate/accelerator.py", line 1851, in _prepare_deepspeed
+    engine, optimizer, _, lr_scheduler = ds_initialize(**kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/__init__.py", line 193, in initialize
+    engine = DeepSpeedEngine(args=args,
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/runtime/engine.py", line 313, in __init__
+    self._configure_optimizer(optimizer, model_parameters)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/runtime/engine.py", line 1302, in _configure_optimizer
+    self.optimizer = self._configure_zero_optimizer(basic_optimizer)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/runtime/engine.py", line 1560, in _configure_zero_optimizer
+    optimizer = DeepSpeedZeroOptimizer(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/runtime/zero/stage_1_and_2.py", line 326, in __init__
+    param.cpu_data = param.data.cpu()
+KeyboardInterrupt
+Error in atexit._run_exitfuncs:
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/ops/transformer/inference/triton/matmul_ext.py", line 27, in is_nfs_path

wandb/run-20241105_162824-9m8ga5sn/files/requirements.txt

@@ -0,0 +1,147 @@
+funcsigs==1.0.2
+sentry-sdk==2.17.0
+multiprocess==0.70.16
+numpy==1.26.2
+pluralizer==1.2.0
+debugpy==1.6.7
+nvidia-cudnn-cu11==8.5.0.96
+deepspeed==0.15.2
+data==0.4
+pandas==2.1.3
+tomli==2.0.1
+charset-normalizer==3.3.2
+attrs==24.2.0
+aiosignal==1.3.1
+fsspec==2023.10.0
+nvidia-cusparse-cu11==11.7.4.91
+zipp==3.12.0
+mypy-extensions==1.0.0
+datasets==3.0.1
+joblib==1.3.2
+hjson==3.1.0
+traitlets==5.7.1
+stack-data==0.6.0
+transformers==4.45.1
+sympy==1.11.1
+Pygments==2.15.0
+docker-pycreds==0.4.0
+dill==0.3.8
+wheel==0.44.0
+prompt-toolkit==3.0.30
+parso==0.8.3
+ipykernel==6.23.1
+pyarrow==17.0.0
+certifi==2023.11.17
+nvidia-cufft-cu11==10.9.0.58
+six==1.16.0
+pydantic==2.9.2
+click==8.1.7
+nest-asyncio==1.5.6
+gmpy2==2.1.0
+matplotlib==3.8.2
+scipy==1.11.4
+typing_extensions==4.12.2
+statsmodels==0.14.0
+huggingface-hub==0.25.0
+frozenlist==1.4.1
+gpustat==1.1.1
+nvidia-nvtx-cu11==11.7.91
+safetensors==0.4.5
+stanza==1.9.2
+decorator==5.1.1
+seaborn==0.13.0
+sentencepiece==0.2.0
+PyYAML==6.0.1
+black==24.8.0
+protobuf==4.25.1
+pickleshare==0.7.5
+peft==0.13.0
+triton==2.0.0
+nvidia-cuda-runtime-cu11==11.7.99
+Jinja2==3.1.2
+nvidia-cusolver-cu11==11.4.0.1
+executing==1.2.0
+jupyter_client==8.1.0
+pluggy==1.3.0
+cmake==3.30.3
+pytz==2023.3.post1
+aiohappyeyeballs==2.4.2
+kiwisolver==1.4.5
+py-cpuinfo==9.0.0
+Pillow==10.1.0
+ptyprocess==0.7.0
+importlib_resources==6.4.5
+GitPython==3.1.43
+importlib-metadata==6.0.0
+iniconfig==2.0.0
+scikit-learn==1.3.2
+exceptiongroup==1.1.0
+networkx==2.8.6
+accelerate==1.0.0
+nltk==3.8.1
+shutilwhich==1.1.0
+fonttools==4.45.1
+future==0.18.3
+aiohttp==3.10.6
+wcwidth==0.2.5
+idna==3.6
+filelock==3.12.2
+pathspec==0.12.1
+jupyter_core==5.1.0
+lit==18.1.8
+nvidia-curand-cu11==10.2.10.91
+nvidia-cublas-cu11==11.10.3.66
+nvidia-ml-py==12.560.30
+msgpack==1.1.0
+python-dateutil==2.8.2
+blessed==1.20.0
+packaging==23.0
+gitdb==4.0.11
+yarl==1.13.0
+emoji==2.8.0
+tzdata==2023.3
+cycler==0.12.1
+tornado==6.2
+backcall==0.2.0
+plotnine==0.12.4
+ninja==1.11.1.1
+latex==0.7.0
+wandb==0.18.5
+setproctitle==1.3.3
+threadpoolctl==3.2.0
+requests==2.32.3
+pyparsing==3.1.1
+smmap==5.0.1
+pyzmq==23.0.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+matplotlib-inline==0.1.6
+latexcodec==1.0.0
+ipython==8.0.0
+patsy==0.5.3
+contourpy==1.2.0
+multidict==6.1.0
+mizani==0.9.3
+urllib3==2.1.0
+tokenizers==0.20.0
+MarkupSafe==2.1.2
+pip==24.2
+pexpect==4.8.0
+tqdm==4.66.5
+jedi==0.18.2
+pydantic_core==2.23.4
+tempdir==0.7.1
+mpmath==1.2.1
+setuptools==72.1.0
+pytest==7.4.3
+pure-eval==0.2.2
+psutil==5.9.1
+comm==0.1.2
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+regex==2023.10.3
+platformdirs==2.5.2
+asttokens==2.2.1
+torch==2.0.0
+nvidia-nccl-cu11==2.14.3
+xxhash==3.5.0

wandb/run-20241105_162824-9m8ga5sn/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-11-05T21:28:24.367518Z",
+  "args":  [
+    "--perturbation",
+    "shuffle_deterministic21",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "3",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1785662992384"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241105_162824-9m8ga5sn/logs/debug-internal.log

@@ -0,0 +1,8 @@
+{"time":"2024-11-05T16:28:24.371628326-05:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-11-05T16:28:24.371640476-05:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_162824-9m8ga5sn/logs/debug-core.log"}
+{"time":"2024-11-05T16:28:24.485562338-05:00","level":"INFO","msg":"created new stream","id":"9m8ga5sn"}
+{"time":"2024-11-05T16:28:24.485608159-05:00","level":"INFO","msg":"stream: started","id":"9m8ga5sn"}
+{"time":"2024-11-05T16:28:24.485700019-05:00","level":"INFO","msg":"sender: started","stream_id":"9m8ga5sn"}
+{"time":"2024-11-05T16:28:24.485694869-05:00","level":"INFO","msg":"handler: started","stream_id":{"value":"9m8ga5sn"}}
+{"time":"2024-11-05T16:28:24.485686569-05:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"9m8ga5sn"}}
+{"time":"2024-11-05T16:28:24.713271003-05:00","level":"INFO","msg":"Starting system monitor"}

wandb/run-20241105_162824-9m8ga5sn/logs/debug.log

@@ -0,0 +1,26 @@
+2024-11-05 16:28:24,365 INFO    MainThread:1777857 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-11-05 16:28:24,365 INFO    MainThread:1777857 [wandb_setup.py:_flush():79] Configure stats pid to 1777857
+2024-11-05 16:28:24,365 INFO    MainThread:1777857 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-11-05 16:28:24,365 INFO    MainThread:1777857 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-11-05 16:28:24,365 INFO    MainThread:1777857 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-11-05 16:28:24,365 INFO    MainThread:1777857 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-11-05 16:28:24,365 INFO    MainThread:1777857 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-11-05 16:28:24,365 INFO    MainThread:1777857 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-11-05 16:28:24,365 INFO    MainThread:1777857 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_162824-9m8ga5sn/logs/debug.log
+2024-11-05 16:28:24,365 INFO    MainThread:1777857 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_162824-9m8ga5sn/logs/debug-internal.log
+2024-11-05 16:28:24,365 INFO    MainThread:1777857 [wandb_init.py:init():621] calling init triggers
+2024-11-05 16:28:24,365 INFO    MainThread:1777857 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-11-05 16:28:24,365 INFO    MainThread:1777857 [wandb_init.py:init():671] starting backend
+2024-11-05 16:28:24,365 INFO    MainThread:1777857 [wandb_init.py:init():675] sending inform_init request
+2024-11-05 16:28:24,366 INFO    MainThread:1777857 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-11-05 16:28:24,367 INFO    MainThread:1777857 [wandb_init.py:init():688] backend started and connected
+2024-11-05 16:28:24,370 INFO    MainThread:1777857 [wandb_init.py:init():783] updated telemetry
+2024-11-05 16:28:24,410 INFO    MainThread:1777857 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-11-05 16:28:24,710 INFO    MainThread:1777857 [wandb_init.py:init():867] starting run threads in backend
+2024-11-05 16:28:24,797 INFO    MainThread:1777857 [wandb_run.py:_console_start():2463] atexit reg
+2024-11-05 16:28:24,797 INFO    MainThread:1777857 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-11-05 16:28:24,797 INFO    MainThread:1777857 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-11-05 16:28:24,798 INFO    MainThread:1777857 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-11-05 16:28:24,799 INFO    MainThread:1777857 [wandb_init.py:init():911] run started, returning control to user process
+2024-11-05 16:28:24,799 INFO    MainThread:1777857 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_deterministic21', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0, 'lr': 5e-06}

wandb/run-20241105_163039-457mt3e1/files/config.yaml

@@ -0,0 +1,49 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.9.19
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "3":
+                - 13
+                - 23
+                - 55
+            "4": 3.9.19
+            "5": 0.18.5
+            "6": 4.45.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+batch_size:
+    value: 3
+epoch:
+    value: 3
+lr:
+    value: 5e-06
+perturbation:
+    value: shuffle_deterministic21
+seed:
+    value: 0
+train_set:
+    value: 10M

wandb/run-20241105_163039-457mt3e1/files/output.log

@@ -0,0 +1,68 @@
+model.safetensors.index.json: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████| 20.9k/20.9k [00:00<00:00, 8.35MB/s]
+model-00001-of-00002.safetensors:  67%|███████████████████████████████████████████████████████████████████████▊                                   | 3.33G/4.97G [01:19<00:38, 42.0MB/s]
+Downloading shards:   0%|                                                                                                                                        | 0/2 [01:19<?, ?it/s]
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/urllib3/response.py", line 712, in _error_catcher
+    yield
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/urllib3/response.py", line 812, in _raw_read
+    data = self._fp_read(amt) if not fp_closed else b""
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/urllib3/response.py", line 797, in _fp_read
+    return self._fp.read(amt) if amt is not None else self._fp.read()
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/http/client.py", line 463, in read
+    n = self.readinto(b)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/http/client.py", line 507, in readinto
+    n = self.fp.readinto(b)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/socket.py", line 704, in readinto
+    return self._sock.recv_into(b)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/ssl.py", line 1275, in recv_into
+    return self.read(nbytes, buffer)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/ssl.py", line 1133, in read
+    return self._sslobj.read(len, buffer)
+KeyboardInterrupt
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 174, in <module>
+    model = AutoModelForCausalLM.from_pretrained(model_name,
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/models/auto/auto_factory.py", line 564, in from_pretrained
+    return model_class.from_pretrained(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/modeling_utils.py", line 3769, in from_pretrained
+    resolved_archive_file, sharded_metadata = get_checkpoint_shard_files(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/utils/hub.py", line 1098, in get_checkpoint_shard_files
+    cached_filename = cached_file(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/utils/hub.py", line 403, in cached_file
+    resolved_file = hf_hub_download(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/utils/_deprecation.py", line 101, in inner_f
+    return f(*args, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn
+    return fn(*args, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1232, in hf_hub_download
+    return _hf_hub_download_to_cache_dir(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1381, in _hf_hub_download_to_cache_dir
+    _download_to_tmp_and_move(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1915, in _download_to_tmp_and_move
+    http_get(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 541, in http_get
+    for chunk in r.iter_content(chunk_size=constants.DOWNLOAD_CHUNK_SIZE):
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/requests/models.py", line 820, in generate
+    yield from self.raw.stream(chunk_size, decode_content=True)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/urllib3/response.py", line 934, in stream
+    data = self.read(amt=amt, decode_content=decode_content)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/urllib3/response.py", line 877, in read
+    data = self._raw_read(amt)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/urllib3/response.py", line 833, in _raw_read
+    raise IncompleteRead(self._fp_bytes_read, self.length_remaining)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/contextlib.py", line 137, in __exit__
+    self.gen.throw(typ, value, traceback)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/urllib3/response.py", line 748, in _error_catcher
+    self._connection.close()
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/urllib3/connection.py", line 272, in close
+    super().close()
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/http/client.py", line 965, in close
+    sock.close()   # close it manually... there may be other refs
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/socket.py", line 501, in close
+    self._real_close()
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/ssl.py", line 1333, in _real_close
+    self._sslobj = None
+KeyboardInterrupt

wandb/run-20241105_163039-457mt3e1/files/requirements.txt

@@ -0,0 +1,147 @@
+funcsigs==1.0.2
+sentry-sdk==2.17.0
+multiprocess==0.70.16
+numpy==1.26.2
+pluralizer==1.2.0
+debugpy==1.6.7
+nvidia-cudnn-cu11==8.5.0.96
+deepspeed==0.15.2
+data==0.4
+pandas==2.1.3
+tomli==2.0.1
+charset-normalizer==3.3.2
+attrs==24.2.0
+aiosignal==1.3.1
+fsspec==2023.10.0
+nvidia-cusparse-cu11==11.7.4.91
+zipp==3.12.0
+mypy-extensions==1.0.0
+datasets==3.0.1
+joblib==1.3.2
+hjson==3.1.0
+traitlets==5.7.1
+stack-data==0.6.0
+transformers==4.45.1
+sympy==1.11.1
+Pygments==2.15.0
+docker-pycreds==0.4.0
+dill==0.3.8
+wheel==0.44.0
+prompt-toolkit==3.0.30
+parso==0.8.3
+ipykernel==6.23.1
+pyarrow==17.0.0
+certifi==2023.11.17
+nvidia-cufft-cu11==10.9.0.58
+six==1.16.0
+pydantic==2.9.2
+click==8.1.7
+nest-asyncio==1.5.6
+gmpy2==2.1.0
+matplotlib==3.8.2
+scipy==1.11.4
+typing_extensions==4.12.2
+statsmodels==0.14.0
+huggingface-hub==0.25.0
+frozenlist==1.4.1
+gpustat==1.1.1
+nvidia-nvtx-cu11==11.7.91
+safetensors==0.4.5
+stanza==1.9.2
+decorator==5.1.1
+seaborn==0.13.0
+sentencepiece==0.2.0
+PyYAML==6.0.1
+black==24.8.0
+protobuf==4.25.1
+pickleshare==0.7.5
+peft==0.13.0
+triton==2.0.0
+nvidia-cuda-runtime-cu11==11.7.99
+Jinja2==3.1.2
+nvidia-cusolver-cu11==11.4.0.1
+executing==1.2.0
+jupyter_client==8.1.0
+pluggy==1.3.0
+cmake==3.30.3
+pytz==2023.3.post1
+aiohappyeyeballs==2.4.2
+kiwisolver==1.4.5
+py-cpuinfo==9.0.0
+Pillow==10.1.0
+ptyprocess==0.7.0
+importlib_resources==6.4.5
+GitPython==3.1.43
+importlib-metadata==6.0.0
+iniconfig==2.0.0
+scikit-learn==1.3.2
+exceptiongroup==1.1.0
+networkx==2.8.6
+accelerate==1.0.0
+nltk==3.8.1
+shutilwhich==1.1.0
+fonttools==4.45.1
+future==0.18.3
+aiohttp==3.10.6
+wcwidth==0.2.5
+idna==3.6
+filelock==3.12.2
+pathspec==0.12.1
+jupyter_core==5.1.0
+lit==18.1.8
+nvidia-curand-cu11==10.2.10.91
+nvidia-cublas-cu11==11.10.3.66
+nvidia-ml-py==12.560.30
+msgpack==1.1.0
+python-dateutil==2.8.2
+blessed==1.20.0
+packaging==23.0
+gitdb==4.0.11
+yarl==1.13.0
+emoji==2.8.0
+tzdata==2023.3
+cycler==0.12.1
+tornado==6.2
+backcall==0.2.0
+plotnine==0.12.4
+ninja==1.11.1.1
+latex==0.7.0
+wandb==0.18.5
+setproctitle==1.3.3
+threadpoolctl==3.2.0
+requests==2.32.3
+pyparsing==3.1.1
+smmap==5.0.1
+pyzmq==23.0.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+matplotlib-inline==0.1.6
+latexcodec==1.0.0
+ipython==8.0.0
+patsy==0.5.3
+contourpy==1.2.0
+multidict==6.1.0
+mizani==0.9.3
+urllib3==2.1.0
+tokenizers==0.20.0
+MarkupSafe==2.1.2
+pip==24.2
+pexpect==4.8.0
+tqdm==4.66.5
+jedi==0.18.2
+pydantic_core==2.23.4
+tempdir==0.7.1
+mpmath==1.2.1
+setuptools==72.1.0
+pytest==7.4.3
+pure-eval==0.2.2
+psutil==5.9.1
+comm==0.1.2
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+regex==2023.10.3
+platformdirs==2.5.2
+asttokens==2.2.1
+torch==2.0.0
+nvidia-nccl-cu11==2.14.3
+xxhash==3.5.0

wandb/run-20241105_163039-457mt3e1/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-11-05T21:30:39.141776Z",
+  "args":  [
+    "--perturbation",
+    "shuffle_deterministic21",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "3",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1785984999424"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241105_163039-457mt3e1/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":81}}

wandb/run-20241105_163039-457mt3e1/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2024-11-05T16:30:39.144372085-05:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-11-05T16:30:39.144400915-05:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_163039-457mt3e1/logs/debug-core.log"}
+{"time":"2024-11-05T16:30:39.251295098-05:00","level":"INFO","msg":"created new stream","id":"457mt3e1"}
+{"time":"2024-11-05T16:30:39.251325458-05:00","level":"INFO","msg":"stream: started","id":"457mt3e1"}
+{"time":"2024-11-05T16:30:39.251426298-05:00","level":"INFO","msg":"sender: started","stream_id":"457mt3e1"}
+{"time":"2024-11-05T16:30:39.251417888-05:00","level":"INFO","msg":"handler: started","stream_id":{"value":"457mt3e1"}}
+{"time":"2024-11-05T16:30:39.251388978-05:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"457mt3e1"}}
+{"time":"2024-11-05T16:30:40.395953842-05:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-11-05T16:32:00.263761688-05:00","level":"INFO","msg":"stream: closing","id":"457mt3e1"}
+{"time":"2024-11-05T16:32:00.263792138-05:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-11-05T16:32:00.264428021-05:00","level":"INFO","msg":"Stopped system monitor"}

wandb/run-20241106_232725-8w3vgdka/files/config.yaml

@@ -0,0 +1,49 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.9.19
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "3":
+                - 13
+                - 23
+                - 55
+            "4": 3.9.19
+            "5": 0.18.5
+            "6": 4.45.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+batch_size:
+    value: 3
+epoch:
+    value: 3
+lr:
+    value: 5e-06
+perturbation:
+    value: shuffle_even_odd
+seed:
+    value: 0
+train_set:
+    value: 10M