Add files using upload-large-folder tool

.gitattributes

@@ -138,3 +138,6 @@ wandb/run-20241031_122113-8ldget07/run-8ldget07.wandb filter=lfs diff=lfs merge=
 wandb/run-20241129_235241-3vsgtb8n/run-3vsgtb8n.wandb filter=lfs diff=lfs merge=lfs -text
 wandb/run-20241106_234111-gz9chn3b/run-gz9chn3b.wandb filter=lfs diff=lfs merge=lfs -text
 wandb/run-20241119_135256-3ybits7j/run-3ybits7j.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241030_222932-48sfxfqm/run-48sfxfqm.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241130_000215-17dv9j80/run-17dv9j80.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241030_222932-9ywqkbmn/run-9ywqkbmn.wandb filter=lfs diff=lfs merge=lfs -text

wandb/run-20241030_011013-ld7lpsap/files/config.yaml

@@ -0,0 +1,47 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.9.19
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "3":
+                - 13
+                - 23
+                - 55
+            "4": 3.9.19
+            "5": 0.18.5
+            "6": 4.45.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+batch_size:
+    value: 3
+epoch:
+    value: 7
+perturbation:
+    value: reverse_control
+seed:
+    value: 0
+train_set:
+    value: 10M

wandb/run-20241030_011013-ld7lpsap/files/output.log

@@ -0,0 +1,6 @@
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 165, in <module>
+    valid_dataset = dataset['validation']
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/dataset_dict.py", line 72, in __getitem__
+    return super().__getitem__(k)
+KeyError: 'validation'

wandb/run-20241030_011013-ld7lpsap/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-10-30T05:10:13.907061Z",
+  "args":  [
+    "--perturbation",
+    "reverse_control",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "7",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1719200362496"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241030_011013-ld7lpsap/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":8}}

wandb/run-20241030_011013-ld7lpsap/logs/debug-internal.log

@@ -0,0 +1,16 @@
+{"time":"2024-10-30T01:10:13.909710076-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-10-30T01:10:13.909728756-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_011013-ld7lpsap/logs/debug-core.log"}
+{"time":"2024-10-30T01:10:14.017126155-04:00","level":"INFO","msg":"created new stream","id":"ld7lpsap"}
+{"time":"2024-10-30T01:10:14.017157555-04:00","level":"INFO","msg":"stream: started","id":"ld7lpsap"}
+{"time":"2024-10-30T01:10:14.017223805-04:00","level":"INFO","msg":"sender: started","stream_id":"ld7lpsap"}
+{"time":"2024-10-30T01:10:14.017175915-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"ld7lpsap"}}
+{"time":"2024-10-30T01:10:14.017224175-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"ld7lpsap"}}
+{"time":"2024-10-30T01:10:14.553170914-04:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-10-30T01:10:21.993916943-04:00","level":"INFO","msg":"stream: closing","id":"ld7lpsap"}
+{"time":"2024-10-30T01:10:21.993954173-04:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-10-30T01:10:21.994592637-04:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-10-30T01:10:22.338001398-04:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-10-30T01:10:22.489878259-04:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"ld7lpsap"}}
+{"time":"2024-10-30T01:10:22.489938679-04:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"ld7lpsap"}}
+{"time":"2024-10-30T01:10:22.48995727-04:00","level":"INFO","msg":"sender: closed","stream_id":"ld7lpsap"}
+{"time":"2024-10-30T01:10:22.49000288-04:00","level":"INFO","msg":"stream: closed","id":"ld7lpsap"}

wandb/run-20241030_011013-ld7lpsap/logs/debug.log

@@ -0,0 +1,27 @@
+2024-10-30 01:10:13,905 INFO    MainThread:323569 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-10-30 01:10:13,905 INFO    MainThread:323569 [wandb_setup.py:_flush():79] Configure stats pid to 323569
+2024-10-30 01:10:13,905 INFO    MainThread:323569 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-10-30 01:10:13,905 INFO    MainThread:323569 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-10-30 01:10:13,905 INFO    MainThread:323569 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-30 01:10:13,905 INFO    MainThread:323569 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-30 01:10:13,905 INFO    MainThread:323569 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-10-30 01:10:13,905 INFO    MainThread:323569 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-30 01:10:13,905 INFO    MainThread:323569 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_011013-ld7lpsap/logs/debug.log
+2024-10-30 01:10:13,905 INFO    MainThread:323569 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_011013-ld7lpsap/logs/debug-internal.log
+2024-10-30 01:10:13,905 INFO    MainThread:323569 [wandb_init.py:init():621] calling init triggers
+2024-10-30 01:10:13,905 INFO    MainThread:323569 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-10-30 01:10:13,905 INFO    MainThread:323569 [wandb_init.py:init():671] starting backend
+2024-10-30 01:10:13,905 INFO    MainThread:323569 [wandb_init.py:init():675] sending inform_init request
+2024-10-30 01:10:13,906 INFO    MainThread:323569 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-30 01:10:13,906 INFO    MainThread:323569 [wandb_init.py:init():688] backend started and connected
+2024-10-30 01:10:13,909 INFO    MainThread:323569 [wandb_init.py:init():783] updated telemetry
+2024-10-30 01:10:13,937 INFO    MainThread:323569 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-10-30 01:10:14,550 INFO    MainThread:323569 [wandb_init.py:init():867] starting run threads in backend
+2024-10-30 01:10:14,643 INFO    MainThread:323569 [wandb_run.py:_console_start():2463] atexit reg
+2024-10-30 01:10:14,644 INFO    MainThread:323569 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-10-30 01:10:14,644 INFO    MainThread:323569 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-10-30 01:10:14,644 INFO    MainThread:323569 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-10-30 01:10:14,645 INFO    MainThread:323569 [wandb_init.py:init():911] run started, returning control to user process
+2024-10-30 01:10:14,645 INFO    MainThread:323569 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_control', 'train_set': '10M', 'batch_size': 3, 'epoch': 7, 'seed': 0}
+2024-10-30 01:10:21,994 WARNING MsgRouterThr:323569 [router.py:message_loop():77] message_loop has been closed

wandb/run-20241030_112700-xnog0orl/files/config.yaml

@@ -0,0 +1,47 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.9.19
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "3":
+                - 13
+                - 23
+                - 55
+            "4": 3.9.19
+            "5": 0.18.5
+            "6": 4.45.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+batch_size:
+    value: 3
+epoch:
+    value: 3
+perturbation:
+    value: reverse_control
+seed:
+    value: 0
+train_set:
+    value: 10M

wandb/run-20241030_112700-xnog0orl/files/output.log

@@ -0,0 +1,34 @@
+Downloading shards:   0%|                                                                                                                                        | 0/2 [01:32<?, ?it/s]
+Error in sys.excepthook:
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/wandb/sdk/lib/exit_hooks.py", line 41, in exc_handler
+    def exc_handler(
+KeyboardInterrupt
+
+Original exception was:
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 172, in <module>
+    model = AutoModelForCausalLM.from_pretrained(model_name,
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/models/auto/auto_factory.py", line 564, in from_pretrained
+    return model_class.from_pretrained(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/modeling_utils.py", line 3769, in from_pretrained
+    resolved_archive_file, sharded_metadata = get_checkpoint_shard_files(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/utils/hub.py", line 1098, in get_checkpoint_shard_files
+    cached_filename = cached_file(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/utils/hub.py", line 403, in cached_file
+    resolved_file = hf_hub_download(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/utils/_deprecation.py", line 101, in inner_f
+    return f(*args, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn
+    return fn(*args, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1232, in hf_hub_download
+    return _hf_hub_download_to_cache_dir(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1380, in _hf_hub_download_to_cache_dir
+    with WeakFileLock(lock_path):
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/contextlib.py", line 119, in __enter__
+    return next(self.gen)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/utils/_fixes.py", line 98, in WeakFileLock
+    lock.acquire()
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/filelock/_api.py", line 225, in acquire
+    time.sleep(poll_interval)
+KeyboardInterrupt

wandb/run-20241030_112700-xnog0orl/files/requirements.txt

@@ -0,0 +1,147 @@
+funcsigs==1.0.2
+sentry-sdk==2.17.0
+multiprocess==0.70.16
+numpy==1.26.2
+pluralizer==1.2.0
+debugpy==1.6.7
+nvidia-cudnn-cu11==8.5.0.96
+deepspeed==0.15.2
+data==0.4
+pandas==2.1.3
+tomli==2.0.1
+charset-normalizer==3.3.2
+attrs==24.2.0
+aiosignal==1.3.1
+fsspec==2023.10.0
+nvidia-cusparse-cu11==11.7.4.91
+zipp==3.12.0
+mypy-extensions==1.0.0
+datasets==3.0.1
+joblib==1.3.2
+hjson==3.1.0
+traitlets==5.7.1
+stack-data==0.6.0
+transformers==4.45.1
+sympy==1.11.1
+Pygments==2.15.0
+docker-pycreds==0.4.0
+dill==0.3.8
+wheel==0.44.0
+prompt-toolkit==3.0.30
+parso==0.8.3
+ipykernel==6.23.1
+pyarrow==17.0.0
+certifi==2023.11.17
+nvidia-cufft-cu11==10.9.0.58
+six==1.16.0
+pydantic==2.9.2
+click==8.1.7
+nest-asyncio==1.5.6
+gmpy2==2.1.0
+matplotlib==3.8.2
+scipy==1.11.4
+typing_extensions==4.12.2
+statsmodels==0.14.0
+huggingface-hub==0.25.0
+frozenlist==1.4.1
+gpustat==1.1.1
+nvidia-nvtx-cu11==11.7.91
+safetensors==0.4.5
+stanza==1.9.2
+decorator==5.1.1
+seaborn==0.13.0
+sentencepiece==0.2.0
+PyYAML==6.0.1
+black==24.8.0
+protobuf==4.25.1
+pickleshare==0.7.5
+peft==0.13.0
+triton==2.0.0
+nvidia-cuda-runtime-cu11==11.7.99
+Jinja2==3.1.2
+nvidia-cusolver-cu11==11.4.0.1
+executing==1.2.0
+jupyter_client==8.1.0
+pluggy==1.3.0
+cmake==3.30.3
+pytz==2023.3.post1
+aiohappyeyeballs==2.4.2
+kiwisolver==1.4.5
+py-cpuinfo==9.0.0
+Pillow==10.1.0
+ptyprocess==0.7.0
+importlib_resources==6.4.5
+GitPython==3.1.43
+importlib-metadata==6.0.0
+iniconfig==2.0.0
+scikit-learn==1.3.2
+exceptiongroup==1.1.0
+networkx==2.8.6
+accelerate==1.0.0
+nltk==3.8.1
+shutilwhich==1.1.0
+fonttools==4.45.1
+future==0.18.3
+aiohttp==3.10.6
+wcwidth==0.2.5
+idna==3.6
+filelock==3.12.2
+pathspec==0.12.1
+jupyter_core==5.1.0
+lit==18.1.8
+nvidia-curand-cu11==10.2.10.91
+nvidia-cublas-cu11==11.10.3.66
+nvidia-ml-py==12.560.30
+msgpack==1.1.0
+python-dateutil==2.8.2
+blessed==1.20.0
+packaging==23.0
+gitdb==4.0.11
+yarl==1.13.0
+emoji==2.8.0
+tzdata==2023.3
+cycler==0.12.1
+tornado==6.2
+backcall==0.2.0
+plotnine==0.12.4
+ninja==1.11.1.1
+latex==0.7.0
+wandb==0.18.5
+setproctitle==1.3.3
+threadpoolctl==3.2.0
+requests==2.32.3
+pyparsing==3.1.1
+smmap==5.0.1
+pyzmq==23.0.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+matplotlib-inline==0.1.6
+latexcodec==1.0.0
+ipython==8.0.0
+patsy==0.5.3
+contourpy==1.2.0
+multidict==6.1.0
+mizani==0.9.3
+urllib3==2.1.0
+tokenizers==0.20.0
+MarkupSafe==2.1.2
+pip==24.2
+pexpect==4.8.0
+tqdm==4.66.5
+jedi==0.18.2
+pydantic_core==2.23.4
+tempdir==0.7.1
+mpmath==1.2.1
+setuptools==72.1.0
+pytest==7.4.3
+pure-eval==0.2.2
+psutil==5.9.1
+comm==0.1.2
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+regex==2023.10.3
+platformdirs==2.5.2
+asttokens==2.2.1
+torch==2.0.0
+nvidia-nccl-cu11==2.14.3
+xxhash==3.5.0

wandb/run-20241030_112700-xnog0orl/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-10-30T15:27:00.647598Z",
+  "args":  [
+    "--perturbation",
+    "reverse_control",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "3",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1710831083520"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241030_112700-xnog0orl/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":93}}

wandb/run-20241030_112700-xnog0orl/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2024-10-30T11:27:00.649532418-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-10-30T11:27:00.649542808-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_112700-xnog0orl/logs/debug-core.log"}
+{"time":"2024-10-30T11:27:00.755411418-04:00","level":"INFO","msg":"created new stream","id":"xnog0orl"}
+{"time":"2024-10-30T11:27:00.755457309-04:00","level":"INFO","msg":"stream: started","id":"xnog0orl"}
+{"time":"2024-10-30T11:27:00.755490549-04:00","level":"INFO","msg":"sender: started","stream_id":"xnog0orl"}
+{"time":"2024-10-30T11:27:00.755467139-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"xnog0orl"}}
+{"time":"2024-10-30T11:27:00.755504459-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"xnog0orl"}}
+{"time":"2024-10-30T11:27:00.992711028-04:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-10-30T11:28:34.162212159-04:00","level":"INFO","msg":"stream: closing","id":"xnog0orl"}
+{"time":"2024-10-30T11:28:34.162280479-04:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-10-30T11:28:34.163422756-04:00","level":"INFO","msg":"Stopped system monitor"}

wandb/run-20241030_112700-xnog0orl/logs/debug.log

@@ -0,0 +1,27 @@
+2024-10-30 11:27:00,644 INFO    MainThread:366802 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-10-30 11:27:00,644 INFO    MainThread:366802 [wandb_setup.py:_flush():79] Configure stats pid to 366802
+2024-10-30 11:27:00,644 INFO    MainThread:366802 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-10-30 11:27:00,644 INFO    MainThread:366802 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-10-30 11:27:00,644 INFO    MainThread:366802 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-30 11:27:00,644 INFO    MainThread:366802 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-30 11:27:00,644 INFO    MainThread:366802 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-10-30 11:27:00,644 INFO    MainThread:366802 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-30 11:27:00,644 INFO    MainThread:366802 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_112700-xnog0orl/logs/debug.log
+2024-10-30 11:27:00,645 INFO    MainThread:366802 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_112700-xnog0orl/logs/debug-internal.log
+2024-10-30 11:27:00,645 INFO    MainThread:366802 [wandb_init.py:init():621] calling init triggers
+2024-10-30 11:27:00,645 INFO    MainThread:366802 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-10-30 11:27:00,645 INFO    MainThread:366802 [wandb_init.py:init():671] starting backend
+2024-10-30 11:27:00,645 INFO    MainThread:366802 [wandb_init.py:init():675] sending inform_init request
+2024-10-30 11:27:00,646 INFO    MainThread:366802 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-30 11:27:00,647 INFO    MainThread:366802 [wandb_init.py:init():688] backend started and connected
+2024-10-30 11:27:00,651 INFO    MainThread:366802 [wandb_init.py:init():783] updated telemetry
+2024-10-30 11:27:00,679 INFO    MainThread:366802 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-10-30 11:27:00,989 INFO    MainThread:366802 [wandb_init.py:init():867] starting run threads in backend
+2024-10-30 11:27:01,086 INFO    MainThread:366802 [wandb_run.py:_console_start():2463] atexit reg
+2024-10-30 11:27:01,086 INFO    MainThread:366802 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-10-30 11:27:01,086 INFO    MainThread:366802 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-10-30 11:27:01,086 INFO    MainThread:366802 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-10-30 11:27:01,087 INFO    MainThread:366802 [wandb_init.py:init():911] run started, returning control to user process
+2024-10-30 11:27:01,088 INFO    MainThread:366802 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_control', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0}
+2024-10-30 11:28:34,162 WARNING MsgRouterThr:366802 [router.py:message_loop():77] message_loop has been closed

wandb/run-20241030_112852-cp23l3oa/files/config.yaml

@@ -0,0 +1,48 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.9.19
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "3":
+                - 2
+                - 13
+                - 23
+                - 55
+            "4": 3.9.19
+            "5": 0.18.5
+            "6": 4.45.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+batch_size:
+    value: 3
+epoch:
+    value: 3
+perturbation:
+    value: reverse_control
+seed:
+    value: 0
+train_set:
+    value: 10M

wandb/run-20241030_112852-cp23l3oa/files/output.log

@@ -0,0 +1,16 @@
+Downloading shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [02:08<00:00, 64.28s/it]
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:04<00:00,  2.42s/it]
+Map: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 18140/18140 [00:48<00:00, 371.75 examples/s]
+tokenized_valid: Dataset({
+    features: ['input_ids', 'attention_mask'],
+    num_rows: 600
+})
+/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
+  warnings.warn(
+[2024-10-30 11:31:56,978] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2024-10-30 11:32:05,091] [INFO] [comm.py:652:init_distributed] cdb=None
+Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination
+Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...
+Loading extension module cpu_adam...
+Time to load cpu_adam op: 4.609123945236206 seconds
+wandb: WARNING Fatal error while uploading data. Some run data will not be synced, but it will still be written to disk. Use `wandb sync` at the end of the run to try uploading.

wandb/run-20241030_112852-cp23l3oa/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-10-30T15:28:52.883976Z",
+  "args":  [
+    "--perturbation",
+    "reverse_control",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "3",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1710831611904"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241030_112852-cp23l3oa/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":23503}}

wandb/run-20241030_112852-cp23l3oa/logs/debug-internal.log

@@ -0,0 +1,107 @@
+{"time":"2024-10-30T11:28:52.88580441-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-10-30T11:28:52.88581319-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_112852-cp23l3oa/logs/debug-core.log"}
+{"time":"2024-10-30T11:28:52.993658057-04:00","level":"INFO","msg":"created new stream","id":"cp23l3oa"}
+{"time":"2024-10-30T11:28:52.993692028-04:00","level":"INFO","msg":"stream: started","id":"cp23l3oa"}
+{"time":"2024-10-30T11:28:52.993781998-04:00","level":"INFO","msg":"sender: started","stream_id":"cp23l3oa"}
+{"time":"2024-10-30T11:28:52.993731128-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"cp23l3oa"}}
+{"time":"2024-10-30T11:28:52.993706028-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"cp23l3oa"}}
+{"time":"2024-10-30T11:28:53.16703486-04:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-10-30T14:02:23.543373941-04:00","level":"ERROR","msg":"HTTP error","status":404,"method":"POST","url":"https://api.wandb.ai/files/yaning1001-dartmouth-college/impossible_llm_reverse/cp23l3oa/file_stream"}
+{"time":"2024-10-30T14:02:23.565073383-04:00","level":"ERROR+4","msg":"filestream: fatal error: filestream: failed to upload: 404 Not Found path=files/yaning1001-dartmouth-college/impossible_llm_reverse/cp23l3oa/file_stream: {\"error\":\"run impossible_llm_reverse/cp23l3oa not found while streaming file\"}"}
+{"time":"2024-10-30T18:00:35.968504841-04:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-10-30T18:00:35.984868772-04:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-10-30T18:00:36.013498887-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:00:36.969538511-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":1.013950589,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:00:38.431464078-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:00:43.189717534-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:00:51.966998431-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:01:08.849337914-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:01:37.005565536-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":61.049971284,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:01:45.472856396-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:02:37.041356072-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":121.08576184,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:02:45.537799395-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:03:37.073019281-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":181.117427829,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:03:45.590739659-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:04:37.102650293-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":241.147059841,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:04:45.648082485-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:05:37.137756332-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":301.18216699,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:05:45.699632187-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:06:37.17149653-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":361.215906838,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:06:45.752175493-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:07:37.200836759-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":421.245234496,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:07:45.815704415-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:08:37.229671186-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":481.274064684,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:08:45.867841546-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:09:37.256775848-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":541.301185775,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:09:45.924328987-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:10:35.956262381-04:00","level":"WARN","msg":"sender: taking a long time","seconds":600.000829089,"work":"WorkRecord(*service_go_proto.Record_Telemetry); Control(connection_id:\"127.0.0.1:57934\")"}
+{"time":"2024-10-30T18:10:37.324610842-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":601.36902034,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:10:45.976036902-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:11:37.358268141-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":661.402675099,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:11:46.027883819-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:12:37.383323034-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":721.427729082,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:12:46.085687295-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:13:37.411060554-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":781.455473042,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:13:46.14107903-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:14:37.446207127-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":841.490615995,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:14:46.19326246-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:15:37.493459811-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":901.537871459,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:15:46.254699294-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:16:37.525025652-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":961.56943723,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:16:46.307408014-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:16:46.307537546-04:00","level":"ERROR","msg":"sender: sendConfig:","error":"api: failed sending: POST https://api.wandb.ai/graphql giving up after 21 attempt(s)"}
+{"time":"2024-10-30T18:16:46.307710978-04:00","level":"INFO","msg":"sender: succeeded after taking longer than expected","seconds":970.352293417,"work":"WorkRecord(*service_go_proto.Record_Telemetry); Control(connection_id:\"127.0.0.1:57934\")"}
+{"time":"2024-10-30T18:16:46.359243514-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:16:46.405726898-04:00","level":"ERROR","msg":"HTTP error","status":404,"method":"POST","url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:16:46.405802219-04:00","level":"ERROR","msg":"runfiles: CreateRunFiles returned error: returned error 404 Not Found: {\"errors\":[{\"message\":\"run impossible_llm_reverse/cp23l3oa not found during createRunFiles\",\"path\":[\"createRunFiles\"]}],\"data\":{\"createRunFiles\":null}}"}
+{"time":"2024-10-30T18:16:48.418321169-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:16:52.683080142-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:17:01.370220939-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:17:20.929252144-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:17:37.556090851-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":51.248040839,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:17:57.913995222-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:18:37.58506308-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":111.277011238,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:18:57.965536752-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:19:37.615430907-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":171.307376935,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:19:58.02218647-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:20:37.642271153-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":231.334215151,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:20:58.082766166-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:21:37.682829283-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":291.374779371,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:21:58.135877334-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:22:37.715388906-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":351.407338644,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:22:58.186847711-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:23:37.748771047-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":411.440719605,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:23:58.237713765-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:24:37.79000021-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":471.481948668,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:24:58.290976542-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:25:37.8258568-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":531.517805047,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:25:58.341705244-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:26:37.858027821-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":591.549972369,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:26:46.308589432-04:00","level":"WARN","msg":"sender: taking a long time","seconds":600.000283177,"work":"WorkRecord(*service_go_proto.Request_Defer); Control(local:true  always_send:true)"}
+{"time":"2024-10-30T18:26:58.398135513-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:27:37.890644202-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":651.5825837,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:27:58.455840564-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:28:37.93601962-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":711.627964587,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:28:58.516600202-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:29:37.968972324-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":771.660914392,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:29:58.589788172-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:30:38.001449018-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":831.693397826,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:30:58.645871422-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:31:38.032170509-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":891.724113647,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:31:58.747420252-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:32:38.062887757-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":951.754833635,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:32:58.797262765-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:32:58.797362996-04:00","level":"ERROR","msg":"sender: sendConfig:","error":"api: failed sending: POST https://api.wandb.ai/graphql giving up after 21 attempt(s)"}
+{"time":"2024-10-30T18:32:58.79775819-04:00","level":"INFO","msg":"sender: succeeded after taking longer than expected","seconds":972.489508325,"work":"WorkRecord(*service_go_proto.Request_Defer); Control(local:true  always_send:true)"}
+{"time":"2024-10-30T18:32:58.894853784-04:00","level":"ERROR","msg":"HTTP error","status":404,"method":"POST","url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:32:58.894958475-04:00","level":"ERROR","msg":"runfiles: CreateRunFiles returned error: returned error 404 Not Found: {\"errors\":[{\"message\":\"run impossible_llm_reverse/cp23l3oa not found during createRunFiles\",\"path\":[\"createRunFiles\"]}],\"data\":{\"createRunFiles\":null}}"}
+{"time":"2024-10-30T18:32:59.007072623-04:00","level":"ERROR","msg":"HTTP error","status":404,"method":"POST","url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:32:59.007367715-04:00","level":"ERROR","msg":"sender: failed to save job artifact: ArtifactSaver.createManifest: returned error 404 Not Found: {\"errors\":[{\"message\":\"failed to find run impossible_llm_reverse/cp23l3oa\",\"path\":[\"createArtifactManifest\"]}],\"data\":{\"createArtifactManifest\":null}}"}
+{"time":"2024-10-30T18:32:59.058985355-04:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-10-30T18:32:59.109609116-04:00","level":"ERROR","msg":"HTTP error","status":404,"method":"POST","url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:32:59.109673806-04:00","level":"ERROR","msg":"runfiles: CreateRunFiles returned error: returned error 404 Not Found: {\"errors\":[{\"message\":\"run impossible_llm_reverse/cp23l3oa not found during createRunFiles\",\"path\":[\"createRunFiles\"]}],\"data\":{\"createRunFiles\":null}}"}
+{"time":"2024-10-30T18:33:00.072305595-04:00","level":"INFO","msg":"stream: closing","id":"cp23l3oa"}
+{"time":"2024-10-30T18:33:00.072351545-04:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"cp23l3oa"}}
+{"time":"2024-10-30T18:33:00.072374975-04:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"cp23l3oa"}}
+{"time":"2024-10-30T18:33:00.072417396-04:00","level":"INFO","msg":"sender: closed","stream_id":"cp23l3oa"}
+{"time":"2024-10-30T18:33:00.072540587-04:00","level":"INFO","msg":"stream: closed","id":"cp23l3oa"}

wandb/run-20241030_112852-cp23l3oa/logs/debug.log

@@ -0,0 +1,33 @@
+2024-10-30 11:28:52,880 INFO    MainThread:367768 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-10-30 11:28:52,880 INFO    MainThread:367768 [wandb_setup.py:_flush():79] Configure stats pid to 367768
+2024-10-30 11:28:52,880 INFO    MainThread:367768 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-10-30 11:28:52,880 INFO    MainThread:367768 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-10-30 11:28:52,880 INFO    MainThread:367768 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-30 11:28:52,880 INFO    MainThread:367768 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-30 11:28:52,880 INFO    MainThread:367768 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-10-30 11:28:52,880 INFO    MainThread:367768 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-30 11:28:52,881 INFO    MainThread:367768 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_112852-cp23l3oa/logs/debug.log
+2024-10-30 11:28:52,881 INFO    MainThread:367768 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_112852-cp23l3oa/logs/debug-internal.log
+2024-10-30 11:28:52,881 INFO    MainThread:367768 [wandb_init.py:init():621] calling init triggers
+2024-10-30 11:28:52,881 INFO    MainThread:367768 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-10-30 11:28:52,881 INFO    MainThread:367768 [wandb_init.py:init():671] starting backend
+2024-10-30 11:28:52,881 INFO    MainThread:367768 [wandb_init.py:init():675] sending inform_init request
+2024-10-30 11:28:52,883 INFO    MainThread:367768 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-30 11:28:52,883 INFO    MainThread:367768 [wandb_init.py:init():688] backend started and connected
+2024-10-30 11:28:52,886 INFO    MainThread:367768 [wandb_init.py:init():783] updated telemetry
+2024-10-30 11:28:52,903 INFO    MainThread:367768 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-10-30 11:28:53,164 INFO    MainThread:367768 [wandb_init.py:init():867] starting run threads in backend
+2024-10-30 11:28:53,261 INFO    MainThread:367768 [wandb_run.py:_console_start():2463] atexit reg
+2024-10-30 11:28:53,261 INFO    MainThread:367768 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-10-30 11:28:53,261 INFO    MainThread:367768 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-10-30 11:28:53,261 INFO    MainThread:367768 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-10-30 11:28:53,263 INFO    MainThread:367768 [wandb_init.py:init():911] run started, returning control to user process
+2024-10-30 11:28:53,263 INFO    MainThread:367768 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_control', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0}
+2024-10-30 18:00:35,945 INFO    MainThread:367768 [wandb_run.py:_finish():2158] finishing run yaning1001-dartmouth-college/impossible_llm_reverse/cp23l3oa
+2024-10-30 18:00:35,955 INFO    MainThread:367768 [wandb_run.py:_atexit_cleanup():2426] got exitcode: 0
+2024-10-30 18:00:35,956 INFO    MainThread:367768 [wandb_run.py:_restore():2408] restore
+2024-10-30 18:00:35,956 INFO    MainThread:367768 [wandb_run.py:_restore():2414] restore done
+2024-10-30 18:33:00,063 INFO    MainThread:367768 [wandb_run.py:_footer_history_summary_info():3975] rendering history
+2024-10-30 18:33:00,063 INFO    MainThread:367768 [wandb_run.py:_footer_history_summary_info():4007] rendering summary
+2024-10-30 18:33:00,071 INFO    MainThread:367768 [wandb_run.py:_footer_sync_info():3934] logging synced files

wandb/run-20241030_222932-48sfxfqm/run-48sfxfqm.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd185388f0883faff5073a5f7e058491253bbef5001577d57dbe9b503f098ca9
+size 1015808

wandb/run-20241030_222932-9ywqkbmn/run-9ywqkbmn.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9d675e24883dca5f9a554545967630053685dd16ec65eedb0e3a3a04b581aad
+size 1048576

wandb/run-20241031_001055-5lnonlg3/files/output.log

@@ -0,0 +1,13 @@
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:05<00:00,  2.90s/it]
+tokenized_valid: Dataset({
+    features: ['input_ids', 'attention_mask'],
+    num_rows: 600
+})
+/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
+  warnings.warn(
+[2024-10-31 00:11:03,904] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2024-10-31 00:11:12,792] [INFO] [comm.py:652:init_distributed] cdb=None
+Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination
+Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...
+Loading extension module cpu_adam...
+Time to load cpu_adam op: 5.37715482711792 seconds

wandb/run-20241031_001055-5lnonlg3/files/requirements.txt

@@ -0,0 +1,147 @@
+funcsigs==1.0.2
+sentry-sdk==2.17.0
+multiprocess==0.70.16
+numpy==1.26.2
+pluralizer==1.2.0
+debugpy==1.6.7
+nvidia-cudnn-cu11==8.5.0.96
+deepspeed==0.15.2
+data==0.4
+pandas==2.1.3
+tomli==2.0.1
+charset-normalizer==3.3.2
+attrs==24.2.0
+aiosignal==1.3.1
+fsspec==2023.10.0
+nvidia-cusparse-cu11==11.7.4.91
+zipp==3.12.0
+mypy-extensions==1.0.0
+datasets==3.0.1
+joblib==1.3.2
+hjson==3.1.0
+traitlets==5.7.1
+stack-data==0.6.0
+transformers==4.45.1
+sympy==1.11.1
+Pygments==2.15.0
+docker-pycreds==0.4.0
+dill==0.3.8
+wheel==0.44.0
+prompt-toolkit==3.0.30
+parso==0.8.3
+ipykernel==6.23.1
+pyarrow==17.0.0
+certifi==2023.11.17
+nvidia-cufft-cu11==10.9.0.58
+six==1.16.0
+pydantic==2.9.2
+click==8.1.7
+nest-asyncio==1.5.6
+gmpy2==2.1.0
+matplotlib==3.8.2
+scipy==1.11.4
+typing_extensions==4.12.2
+statsmodels==0.14.0
+huggingface-hub==0.25.0
+frozenlist==1.4.1
+gpustat==1.1.1
+nvidia-nvtx-cu11==11.7.91
+safetensors==0.4.5
+stanza==1.9.2
+decorator==5.1.1
+seaborn==0.13.0
+sentencepiece==0.2.0
+PyYAML==6.0.1
+black==24.8.0
+protobuf==4.25.1
+pickleshare==0.7.5
+peft==0.13.0
+triton==2.0.0
+nvidia-cuda-runtime-cu11==11.7.99
+Jinja2==3.1.2
+nvidia-cusolver-cu11==11.4.0.1
+executing==1.2.0
+jupyter_client==8.1.0
+pluggy==1.3.0
+cmake==3.30.3
+pytz==2023.3.post1
+aiohappyeyeballs==2.4.2
+kiwisolver==1.4.5
+py-cpuinfo==9.0.0
+Pillow==10.1.0
+ptyprocess==0.7.0
+importlib_resources==6.4.5
+GitPython==3.1.43
+importlib-metadata==6.0.0
+iniconfig==2.0.0
+scikit-learn==1.3.2
+exceptiongroup==1.1.0
+networkx==2.8.6
+accelerate==1.0.0
+nltk==3.8.1
+shutilwhich==1.1.0
+fonttools==4.45.1
+future==0.18.3
+aiohttp==3.10.6
+wcwidth==0.2.5
+idna==3.6
+filelock==3.12.2
+pathspec==0.12.1
+jupyter_core==5.1.0
+lit==18.1.8
+nvidia-curand-cu11==10.2.10.91
+nvidia-cublas-cu11==11.10.3.66
+nvidia-ml-py==12.560.30
+msgpack==1.1.0
+python-dateutil==2.8.2
+blessed==1.20.0
+packaging==23.0
+gitdb==4.0.11
+yarl==1.13.0
+emoji==2.8.0
+tzdata==2023.3
+cycler==0.12.1
+tornado==6.2
+backcall==0.2.0
+plotnine==0.12.4
+ninja==1.11.1.1
+latex==0.7.0
+wandb==0.18.5
+setproctitle==1.3.3
+threadpoolctl==3.2.0
+requests==2.32.3
+pyparsing==3.1.1
+smmap==5.0.1
+pyzmq==23.0.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+matplotlib-inline==0.1.6
+latexcodec==1.0.0
+ipython==8.0.0
+patsy==0.5.3
+contourpy==1.2.0
+multidict==6.1.0
+mizani==0.9.3
+urllib3==2.1.0
+tokenizers==0.20.0
+MarkupSafe==2.1.2
+pip==24.2
+pexpect==4.8.0
+tqdm==4.66.5
+jedi==0.18.2
+pydantic_core==2.23.4
+tempdir==0.7.1
+mpmath==1.2.1
+setuptools==72.1.0
+pytest==7.4.3
+pure-eval==0.2.2
+psutil==5.9.1
+comm==0.1.2
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+regex==2023.10.3
+platformdirs==2.5.2
+asttokens==2.2.1
+torch==2.0.0
+nvidia-nccl-cu11==2.14.3
+xxhash==3.5.0

wandb/run-20241031_001055-5lnonlg3/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-10-31T04:10:55.992950Z",
+  "args":  [
+    "--perturbation",
+    "reverse_full",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "6",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1728856920064"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241031_001055-5lnonlg3/logs/debug-internal.log

@@ -0,0 +1,8 @@
+{"time":"2024-10-31T00:10:55.995246959-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-10-31T00:10:55.995268499-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_001055-5lnonlg3/logs/debug-core.log"}
+{"time":"2024-10-31T00:10:56.102481822-04:00","level":"INFO","msg":"created new stream","id":"5lnonlg3"}
+{"time":"2024-10-31T00:10:56.102510232-04:00","level":"INFO","msg":"stream: started","id":"5lnonlg3"}
+{"time":"2024-10-31T00:10:56.102564502-04:00","level":"INFO","msg":"sender: started","stream_id":"5lnonlg3"}
+{"time":"2024-10-31T00:10:56.102549732-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"5lnonlg3"}}
+{"time":"2024-10-31T00:10:56.102573972-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"5lnonlg3"}}
+{"time":"2024-10-31T00:10:56.319490029-04:00","level":"INFO","msg":"Starting system monitor"}

wandb/run-20241031_001055-5lnonlg3/logs/debug.log

@@ -0,0 +1,26 @@
+2024-10-31 00:10:55,990 INFO    MainThread:479387 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-10-31 00:10:55,990 INFO    MainThread:479387 [wandb_setup.py:_flush():79] Configure stats pid to 479387
+2024-10-31 00:10:55,990 INFO    MainThread:479387 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-10-31 00:10:55,990 INFO    MainThread:479387 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-10-31 00:10:55,990 INFO    MainThread:479387 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-31 00:10:55,990 INFO    MainThread:479387 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-31 00:10:55,990 INFO    MainThread:479387 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-10-31 00:10:55,990 INFO    MainThread:479387 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-31 00:10:55,990 INFO    MainThread:479387 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_001055-5lnonlg3/logs/debug.log
+2024-10-31 00:10:55,990 INFO    MainThread:479387 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_001055-5lnonlg3/logs/debug-internal.log
+2024-10-31 00:10:55,990 INFO    MainThread:479387 [wandb_init.py:init():621] calling init triggers
+2024-10-31 00:10:55,990 INFO    MainThread:479387 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-10-31 00:10:55,990 INFO    MainThread:479387 [wandb_init.py:init():671] starting backend
+2024-10-31 00:10:55,990 INFO    MainThread:479387 [wandb_init.py:init():675] sending inform_init request
+2024-10-31 00:10:55,992 INFO    MainThread:479387 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-31 00:10:55,992 INFO    MainThread:479387 [wandb_init.py:init():688] backend started and connected
+2024-10-31 00:10:55,995 INFO    MainThread:479387 [wandb_init.py:init():783] updated telemetry
+2024-10-31 00:10:56,025 INFO    MainThread:479387 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-10-31 00:10:56,315 INFO    MainThread:479387 [wandb_init.py:init():867] starting run threads in backend
+2024-10-31 00:10:56,418 INFO    MainThread:479387 [wandb_run.py:_console_start():2463] atexit reg
+2024-10-31 00:10:56,418 INFO    MainThread:479387 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-10-31 00:10:56,418 INFO    MainThread:479387 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-10-31 00:10:56,418 INFO    MainThread:479387 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-10-31 00:10:56,419 INFO    MainThread:479387 [wandb_init.py:init():911] run started, returning control to user process
+2024-10-31 00:10:56,420 INFO    MainThread:479387 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_full', 'train_set': '10M', 'batch_size': 3, 'epoch': 6, 'seed': 0, 'lr': 1e-05}

wandb/run-20241031_114700-xnsmnnpa/files/output.log

@@ -0,0 +1,41 @@
+Downloading shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [02:32<00:00, 76.41s/it]
+Loading checkpoint shards: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:04<00:00,  2.40s/it]
+tokenized_valid: Dataset({
+    features: ['input_ids', 'attention_mask'],
+    num_rows: 600
+})
+/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
+  warnings.warn(
+[2024-10-31 11:49:40,285] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2024-10-31 11:49:48,756] [INFO] [comm.py:652:init_distributed] cdb=None
+Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination
+Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...
+Loading extension module cpu_adam...
+Time to load cpu_adam op: 4.9282543659210205 seconds
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 220, in <module>
+    trainer.train()
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/trainer.py", line 2052, in train
+    return inner_training_loop(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/trainer.py", line 2388, in _inner_training_loop
+    tr_loss_step = self.training_step(model, inputs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/trainer.py", line 3518, in training_step
+    self.accelerator.backward(loss, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/accelerate/accelerator.py", line 2238, in backward
+    self.deepspeed_engine_wrapped.backward(loss, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/accelerate/utils/deepspeed.py", line 186, in backward
+    self.engine.backward(loss, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/utils/nvtx.py", line 18, in wrapped_fn
+    ret_val = func(*args, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/runtime/engine.py", line 2020, in backward
+    self.optimizer.backward(loss, retain_graph=retain_graph)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/runtime/zero/stage_1_and_2.py", line 2063, in backward
+    self.loss_scaler.backward(loss.float(), retain_graph=retain_graph)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/runtime/fp16/loss_scaler.py", line 63, in backward
+    scaled_loss.backward(retain_graph=retain_graph)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/torch/_tensor.py", line 487, in backward
+    torch.autograd.backward(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/torch/autograd/__init__.py", line 200, in backward
+    Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
+KeyboardInterrupt
+Error in atexit._run_exitfuncs:

wandb/run-20241031_114700-xnsmnnpa/files/requirements.txt

@@ -0,0 +1,147 @@
+funcsigs==1.0.2
+sentry-sdk==2.17.0
+multiprocess==0.70.16
+numpy==1.26.2
+pluralizer==1.2.0
+debugpy==1.6.7
+nvidia-cudnn-cu11==8.5.0.96
+deepspeed==0.15.2
+data==0.4
+pandas==2.1.3
+tomli==2.0.1
+charset-normalizer==3.3.2
+attrs==24.2.0
+aiosignal==1.3.1
+fsspec==2023.10.0
+nvidia-cusparse-cu11==11.7.4.91
+zipp==3.12.0
+mypy-extensions==1.0.0
+datasets==3.0.1
+joblib==1.3.2
+hjson==3.1.0
+traitlets==5.7.1
+stack-data==0.6.0
+transformers==4.45.1
+sympy==1.11.1
+Pygments==2.15.0
+docker-pycreds==0.4.0
+dill==0.3.8
+wheel==0.44.0
+prompt-toolkit==3.0.30
+parso==0.8.3
+ipykernel==6.23.1
+pyarrow==17.0.0
+certifi==2023.11.17
+nvidia-cufft-cu11==10.9.0.58
+six==1.16.0
+pydantic==2.9.2
+click==8.1.7
+nest-asyncio==1.5.6
+gmpy2==2.1.0
+matplotlib==3.8.2
+scipy==1.11.4
+typing_extensions==4.12.2
+statsmodels==0.14.0
+huggingface-hub==0.25.0
+frozenlist==1.4.1
+gpustat==1.1.1
+nvidia-nvtx-cu11==11.7.91
+safetensors==0.4.5
+stanza==1.9.2
+decorator==5.1.1
+seaborn==0.13.0
+sentencepiece==0.2.0
+PyYAML==6.0.1
+black==24.8.0
+protobuf==4.25.1
+pickleshare==0.7.5
+peft==0.13.0
+triton==2.0.0
+nvidia-cuda-runtime-cu11==11.7.99
+Jinja2==3.1.2
+nvidia-cusolver-cu11==11.4.0.1
+executing==1.2.0
+jupyter_client==8.1.0
+pluggy==1.3.0
+cmake==3.30.3
+pytz==2023.3.post1
+aiohappyeyeballs==2.4.2
+kiwisolver==1.4.5
+py-cpuinfo==9.0.0
+Pillow==10.1.0
+ptyprocess==0.7.0
+importlib_resources==6.4.5
+GitPython==3.1.43
+importlib-metadata==6.0.0
+iniconfig==2.0.0
+scikit-learn==1.3.2
+exceptiongroup==1.1.0
+networkx==2.8.6
+accelerate==1.0.0
+nltk==3.8.1
+shutilwhich==1.1.0
+fonttools==4.45.1
+future==0.18.3
+aiohttp==3.10.6
+wcwidth==0.2.5
+idna==3.6
+filelock==3.12.2
+pathspec==0.12.1
+jupyter_core==5.1.0
+lit==18.1.8
+nvidia-curand-cu11==10.2.10.91
+nvidia-cublas-cu11==11.10.3.66
+nvidia-ml-py==12.560.30
+msgpack==1.1.0
+python-dateutil==2.8.2
+blessed==1.20.0
+packaging==23.0
+gitdb==4.0.11
+yarl==1.13.0
+emoji==2.8.0
+tzdata==2023.3
+cycler==0.12.1
+tornado==6.2
+backcall==0.2.0
+plotnine==0.12.4
+ninja==1.11.1.1
+latex==0.7.0
+wandb==0.18.5
+setproctitle==1.3.3
+threadpoolctl==3.2.0
+requests==2.32.3
+pyparsing==3.1.1
+smmap==5.0.1
+pyzmq==23.0.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+matplotlib-inline==0.1.6
+latexcodec==1.0.0
+ipython==8.0.0
+patsy==0.5.3
+contourpy==1.2.0
+multidict==6.1.0
+mizani==0.9.3
+urllib3==2.1.0
+tokenizers==0.20.0
+MarkupSafe==2.1.2
+pip==24.2
+pexpect==4.8.0
+tqdm==4.66.5
+jedi==0.18.2
+pydantic_core==2.23.4
+tempdir==0.7.1
+mpmath==1.2.1
+setuptools==72.1.0
+pytest==7.4.3
+pure-eval==0.2.2
+psutil==5.9.1
+comm==0.1.2
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+regex==2023.10.3
+platformdirs==2.5.2
+asttokens==2.2.1
+torch==2.0.0
+nvidia-nccl-cu11==2.14.3
+xxhash==3.5.0

wandb/run-20241031_114700-xnsmnnpa/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-10-31T15:47:00.194426Z",
+  "args":  [
+    "--perturbation",
+    "reverse_full",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "6",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1753158594560"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241031_114700-xnsmnnpa/logs/debug-internal.log

@@ -0,0 +1,8 @@
+{"time":"2024-10-31T11:47:00.198821275-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-10-31T11:47:00.198840765-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_114700-xnsmnnpa/logs/debug-core.log"}
+{"time":"2024-10-31T11:47:00.310215718-04:00","level":"INFO","msg":"created new stream","id":"xnsmnnpa"}
+{"time":"2024-10-31T11:47:00.310244338-04:00","level":"INFO","msg":"stream: started","id":"xnsmnnpa"}
+{"time":"2024-10-31T11:47:00.310292438-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"xnsmnnpa"}}
+{"time":"2024-10-31T11:47:00.3105868-04:00","level":"INFO","msg":"sender: started","stream_id":"xnsmnnpa"}
+{"time":"2024-10-31T11:47:00.31060146-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"xnsmnnpa"}}
+{"time":"2024-10-31T11:47:00.515719225-04:00","level":"INFO","msg":"Starting system monitor"}

wandb/run-20241031_114700-xnsmnnpa/logs/debug.log

@@ -0,0 +1,26 @@
+2024-10-31 11:47:00,191 INFO    MainThread:554146 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-10-31 11:47:00,192 INFO    MainThread:554146 [wandb_setup.py:_flush():79] Configure stats pid to 554146
+2024-10-31 11:47:00,192 INFO    MainThread:554146 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-10-31 11:47:00,192 INFO    MainThread:554146 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-10-31 11:47:00,192 INFO    MainThread:554146 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-31 11:47:00,192 INFO    MainThread:554146 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-31 11:47:00,192 INFO    MainThread:554146 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-10-31 11:47:00,192 INFO    MainThread:554146 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-31 11:47:00,192 INFO    MainThread:554146 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_114700-xnsmnnpa/logs/debug.log
+2024-10-31 11:47:00,192 INFO    MainThread:554146 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_114700-xnsmnnpa/logs/debug-internal.log
+2024-10-31 11:47:00,192 INFO    MainThread:554146 [wandb_init.py:init():621] calling init triggers
+2024-10-31 11:47:00,192 INFO    MainThread:554146 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-10-31 11:47:00,192 INFO    MainThread:554146 [wandb_init.py:init():671] starting backend
+2024-10-31 11:47:00,192 INFO    MainThread:554146 [wandb_init.py:init():675] sending inform_init request
+2024-10-31 11:47:00,193 INFO    MainThread:554146 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-31 11:47:00,194 INFO    MainThread:554146 [wandb_init.py:init():688] backend started and connected
+2024-10-31 11:47:00,197 INFO    MainThread:554146 [wandb_init.py:init():783] updated telemetry
+2024-10-31 11:47:00,223 INFO    MainThread:554146 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-10-31 11:47:00,511 INFO    MainThread:554146 [wandb_init.py:init():867] starting run threads in backend
+2024-10-31 11:47:00,638 INFO    MainThread:554146 [wandb_run.py:_console_start():2463] atexit reg
+2024-10-31 11:47:00,638 INFO    MainThread:554146 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-10-31 11:47:00,638 INFO    MainThread:554146 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-10-31 11:47:00,638 INFO    MainThread:554146 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-10-31 11:47:00,640 INFO    MainThread:554146 [wandb_init.py:init():911] run started, returning control to user process
+2024-10-31 11:47:00,640 INFO    MainThread:554146 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_full', 'train_set': '10M', 'batch_size': 3, 'epoch': 6, 'seed': 0, 'lr': 0.0001}

wandb/run-20241101_201707-39c9svl8/files/output.log

@@ -0,0 +1,13 @@
+Loading checkpoint shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:04<00:00,  2.44s/it]
+tokenized_valid: Dataset({
+    features: ['input_ids', 'attention_mask'],
+    num_rows: 600
+})
+/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
+  warnings.warn(
+[2024-11-01 20:17:16,614] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2024-11-01 20:17:25,868] [INFO] [comm.py:652:init_distributed] cdb=None
+Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination
+Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...
+Loading extension module cpu_adam...
+Time to load cpu_adam op: 4.978870391845703 seconds

wandb/run-20241101_201707-39c9svl8/files/requirements.txt

@@ -0,0 +1,147 @@
+funcsigs==1.0.2
+sentry-sdk==2.17.0
+multiprocess==0.70.16
+numpy==1.26.2
+pluralizer==1.2.0
+debugpy==1.6.7
+nvidia-cudnn-cu11==8.5.0.96
+deepspeed==0.15.2
+data==0.4
+pandas==2.1.3
+tomli==2.0.1
+charset-normalizer==3.3.2
+attrs==24.2.0
+aiosignal==1.3.1
+fsspec==2023.10.0
+nvidia-cusparse-cu11==11.7.4.91
+zipp==3.12.0
+mypy-extensions==1.0.0
+datasets==3.0.1
+joblib==1.3.2
+hjson==3.1.0
+traitlets==5.7.1
+stack-data==0.6.0
+transformers==4.45.1
+sympy==1.11.1
+Pygments==2.15.0
+docker-pycreds==0.4.0
+dill==0.3.8
+wheel==0.44.0
+prompt-toolkit==3.0.30
+parso==0.8.3
+ipykernel==6.23.1
+pyarrow==17.0.0
+certifi==2023.11.17
+nvidia-cufft-cu11==10.9.0.58
+six==1.16.0
+pydantic==2.9.2
+click==8.1.7
+nest-asyncio==1.5.6
+gmpy2==2.1.0
+matplotlib==3.8.2
+scipy==1.11.4
+typing_extensions==4.12.2
+statsmodels==0.14.0
+huggingface-hub==0.25.0
+frozenlist==1.4.1
+gpustat==1.1.1
+nvidia-nvtx-cu11==11.7.91
+safetensors==0.4.5
+stanza==1.9.2
+decorator==5.1.1
+seaborn==0.13.0
+sentencepiece==0.2.0
+PyYAML==6.0.1
+black==24.8.0
+protobuf==4.25.1
+pickleshare==0.7.5
+peft==0.13.0
+triton==2.0.0
+nvidia-cuda-runtime-cu11==11.7.99
+Jinja2==3.1.2
+nvidia-cusolver-cu11==11.4.0.1
+executing==1.2.0
+jupyter_client==8.1.0
+pluggy==1.3.0
+cmake==3.30.3
+pytz==2023.3.post1
+aiohappyeyeballs==2.4.2
+kiwisolver==1.4.5
+py-cpuinfo==9.0.0
+Pillow==10.1.0
+ptyprocess==0.7.0
+importlib_resources==6.4.5
+GitPython==3.1.43
+importlib-metadata==6.0.0
+iniconfig==2.0.0
+scikit-learn==1.3.2
+exceptiongroup==1.1.0
+networkx==2.8.6
+accelerate==1.0.0
+nltk==3.8.1
+shutilwhich==1.1.0
+fonttools==4.45.1
+future==0.18.3
+aiohttp==3.10.6
+wcwidth==0.2.5
+idna==3.6
+filelock==3.12.2
+pathspec==0.12.1
+jupyter_core==5.1.0
+lit==18.1.8
+nvidia-curand-cu11==10.2.10.91
+nvidia-cublas-cu11==11.10.3.66
+nvidia-ml-py==12.560.30
+msgpack==1.1.0
+python-dateutil==2.8.2
+blessed==1.20.0
+packaging==23.0
+gitdb==4.0.11
+yarl==1.13.0
+emoji==2.8.0
+tzdata==2023.3
+cycler==0.12.1
+tornado==6.2
+backcall==0.2.0
+plotnine==0.12.4
+ninja==1.11.1.1
+latex==0.7.0
+wandb==0.18.5
+setproctitle==1.3.3
+threadpoolctl==3.2.0
+requests==2.32.3
+pyparsing==3.1.1
+smmap==5.0.1
+pyzmq==23.0.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+matplotlib-inline==0.1.6
+latexcodec==1.0.0
+ipython==8.0.0
+patsy==0.5.3
+contourpy==1.2.0
+multidict==6.1.0
+mizani==0.9.3
+urllib3==2.1.0
+tokenizers==0.20.0
+MarkupSafe==2.1.2
+pip==24.2
+pexpect==4.8.0
+tqdm==4.66.5
+jedi==0.18.2
+pydantic_core==2.23.4
+tempdir==0.7.1
+mpmath==1.2.1
+setuptools==72.1.0
+pytest==7.4.3
+pure-eval==0.2.2
+psutil==5.9.1
+comm==0.1.2
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+regex==2023.10.3
+platformdirs==2.5.2
+asttokens==2.2.1
+torch==2.0.0
+nvidia-nccl-cu11==2.14.3
+xxhash==3.5.0

wandb/run-20241101_201707-39c9svl8/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-11-02T00:17:07.965008Z",
+  "args":  [
+    "--perturbation",
+    "shuffle_nondeterministic",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "3",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1754802659328"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241101_201707-39c9svl8/logs/debug-internal.log

@@ -0,0 +1,8 @@
+{"time":"2024-11-01T20:17:07.966195773-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-11-01T20:17:07.966207283-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_201707-39c9svl8/logs/debug-core.log"}
+{"time":"2024-11-01T20:17:08.072934618-04:00","level":"INFO","msg":"created new stream","id":"39c9svl8"}
+{"time":"2024-11-01T20:17:08.072999428-04:00","level":"INFO","msg":"stream: started","id":"39c9svl8"}
+{"time":"2024-11-01T20:17:08.073061239-04:00","level":"INFO","msg":"sender: started","stream_id":"39c9svl8"}
+{"time":"2024-11-01T20:17:08.073039518-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"39c9svl8"}}
+{"time":"2024-11-01T20:17:08.073058889-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"39c9svl8"}}
+{"time":"2024-11-01T20:17:09.97487612-04:00","level":"INFO","msg":"Starting system monitor"}

wandb/run-20241101_201707-39c9svl8/logs/debug.log

@@ -0,0 +1,26 @@
+2024-11-01 20:17:07,962 INFO    MainThread:875619 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-11-01 20:17:07,962 INFO    MainThread:875619 [wandb_setup.py:_flush():79] Configure stats pid to 875619
+2024-11-01 20:17:07,962 INFO    MainThread:875619 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-11-01 20:17:07,963 INFO    MainThread:875619 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-11-01 20:17:07,963 INFO    MainThread:875619 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-11-01 20:17:07,963 INFO    MainThread:875619 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-11-01 20:17:07,963 INFO    MainThread:875619 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-11-01 20:17:07,963 INFO    MainThread:875619 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-11-01 20:17:07,963 INFO    MainThread:875619 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_201707-39c9svl8/logs/debug.log
+2024-11-01 20:17:07,963 INFO    MainThread:875619 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_201707-39c9svl8/logs/debug-internal.log
+2024-11-01 20:17:07,963 INFO    MainThread:875619 [wandb_init.py:init():621] calling init triggers
+2024-11-01 20:17:07,963 INFO    MainThread:875619 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-11-01 20:17:07,963 INFO    MainThread:875619 [wandb_init.py:init():671] starting backend
+2024-11-01 20:17:07,963 INFO    MainThread:875619 [wandb_init.py:init():675] sending inform_init request
+2024-11-01 20:17:07,964 INFO    MainThread:875619 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-11-01 20:17:07,964 INFO    MainThread:875619 [wandb_init.py:init():688] backend started and connected
+2024-11-01 20:17:07,967 INFO    MainThread:875619 [wandb_init.py:init():783] updated telemetry
+2024-11-01 20:17:07,988 INFO    MainThread:875619 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-11-01 20:17:09,971 INFO    MainThread:875619 [wandb_init.py:init():867] starting run threads in backend
+2024-11-01 20:17:10,063 INFO    MainThread:875619 [wandb_run.py:_console_start():2463] atexit reg
+2024-11-01 20:17:10,063 INFO    MainThread:875619 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-11-01 20:17:10,063 INFO    MainThread:875619 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-11-01 20:17:10,063 INFO    MainThread:875619 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-11-01 20:17:10,065 INFO    MainThread:875619 [wandb_init.py:init():911] run started, returning control to user process
+2024-11-01 20:17:10,065 INFO    MainThread:875619 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_nondeterministic', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0, 'lr': 5e-06}

wandb/run-20241101_201910-b6bwhpyd/files/requirements.txt

@@ -0,0 +1,147 @@
+funcsigs==1.0.2
+sentry-sdk==2.17.0
+multiprocess==0.70.16
+numpy==1.26.2
+pluralizer==1.2.0
+debugpy==1.6.7
+nvidia-cudnn-cu11==8.5.0.96
+deepspeed==0.15.2
+data==0.4
+pandas==2.1.3
+tomli==2.0.1
+charset-normalizer==3.3.2
+attrs==24.2.0
+aiosignal==1.3.1
+fsspec==2023.10.0
+nvidia-cusparse-cu11==11.7.4.91
+zipp==3.12.0
+mypy-extensions==1.0.0
+datasets==3.0.1
+joblib==1.3.2
+hjson==3.1.0
+traitlets==5.7.1
+stack-data==0.6.0
+transformers==4.45.1
+sympy==1.11.1
+Pygments==2.15.0
+docker-pycreds==0.4.0
+dill==0.3.8
+wheel==0.44.0
+prompt-toolkit==3.0.30
+parso==0.8.3
+ipykernel==6.23.1
+pyarrow==17.0.0
+certifi==2023.11.17
+nvidia-cufft-cu11==10.9.0.58
+six==1.16.0
+pydantic==2.9.2
+click==8.1.7
+nest-asyncio==1.5.6
+gmpy2==2.1.0
+matplotlib==3.8.2
+scipy==1.11.4
+typing_extensions==4.12.2
+statsmodels==0.14.0
+huggingface-hub==0.25.0
+frozenlist==1.4.1
+gpustat==1.1.1
+nvidia-nvtx-cu11==11.7.91
+safetensors==0.4.5
+stanza==1.9.2
+decorator==5.1.1
+seaborn==0.13.0
+sentencepiece==0.2.0
+PyYAML==6.0.1
+black==24.8.0
+protobuf==4.25.1
+pickleshare==0.7.5
+peft==0.13.0
+triton==2.0.0
+nvidia-cuda-runtime-cu11==11.7.99
+Jinja2==3.1.2
+nvidia-cusolver-cu11==11.4.0.1
+executing==1.2.0
+jupyter_client==8.1.0
+pluggy==1.3.0
+cmake==3.30.3
+pytz==2023.3.post1
+aiohappyeyeballs==2.4.2
+kiwisolver==1.4.5
+py-cpuinfo==9.0.0
+Pillow==10.1.0
+ptyprocess==0.7.0
+importlib_resources==6.4.5
+GitPython==3.1.43
+importlib-metadata==6.0.0
+iniconfig==2.0.0
+scikit-learn==1.3.2
+exceptiongroup==1.1.0
+networkx==2.8.6
+accelerate==1.0.0
+nltk==3.8.1
+shutilwhich==1.1.0
+fonttools==4.45.1
+future==0.18.3
+aiohttp==3.10.6
+wcwidth==0.2.5
+idna==3.6
+filelock==3.12.2
+pathspec==0.12.1
+jupyter_core==5.1.0
+lit==18.1.8
+nvidia-curand-cu11==10.2.10.91
+nvidia-cublas-cu11==11.10.3.66
+nvidia-ml-py==12.560.30
+msgpack==1.1.0
+python-dateutil==2.8.2
+blessed==1.20.0
+packaging==23.0
+gitdb==4.0.11
+yarl==1.13.0
+emoji==2.8.0
+tzdata==2023.3
+cycler==0.12.1
+tornado==6.2
+backcall==0.2.0
+plotnine==0.12.4
+ninja==1.11.1.1
+latex==0.7.0
+wandb==0.18.5
+setproctitle==1.3.3
+threadpoolctl==3.2.0
+requests==2.32.3
+pyparsing==3.1.1
+smmap==5.0.1
+pyzmq==23.0.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+matplotlib-inline==0.1.6
+latexcodec==1.0.0
+ipython==8.0.0
+patsy==0.5.3
+contourpy==1.2.0
+multidict==6.1.0
+mizani==0.9.3
+urllib3==2.1.0
+tokenizers==0.20.0
+MarkupSafe==2.1.2
+pip==24.2
+pexpect==4.8.0
+tqdm==4.66.5
+jedi==0.18.2
+pydantic_core==2.23.4
+tempdir==0.7.1
+mpmath==1.2.1
+setuptools==72.1.0
+pytest==7.4.3
+pure-eval==0.2.2
+psutil==5.9.1
+comm==0.1.2
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+regex==2023.10.3
+platformdirs==2.5.2
+asttokens==2.2.1
+torch==2.0.0
+nvidia-nccl-cu11==2.14.3
+xxhash==3.5.0

wandb/run-20241101_201910-b6bwhpyd/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-11-02T00:19:10.248432Z",
+  "args":  [
+    "--perturbation",
+    "shuffle_nondeterministic",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "3",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1754803580928"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241105_160059-7almhy5u/files/config.yaml

@@ -0,0 +1,49 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.9.19
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "3":
+                - 13
+                - 23
+                - 55
+            "4": 3.9.19
+            "5": 0.18.5
+            "6": 4.45.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+batch_size:
+    value: 3
+epoch:
+    value: 3
+lr:
+    value: 5e-06
+perturbation:
+    value: shuffle_deterministic21
+seed:
+    value: 0
+train_set:
+    value: 10M

wandb/run-20241105_160059-7almhy5u/files/output.log

@@ -0,0 +1,19 @@
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/pathlib.py", line 1323, in mkdir
+    self._accessor.mkdir(self, mode)
+FileNotFoundError: [Errno 2] No such file or directory: '/home/chunhui/.cache/huggingface/datasets/babylm_dataset_test/babylm_shuffle_deterministic21_10M_seed0/0.0.0'
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 165, in <module>
+    dataset = load_dataset('babylm_dataset_test.py', name=dataset_name, trust_remote_code=True)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/load.py", line 2096, in load_dataset
+    builder_instance.download_and_prepare(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/builder.py", line 855, in download_and_prepare
+    Path(self._output_dir).parent.mkdir(parents=True, exist_ok=True)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/pathlib.py", line 1327, in mkdir
+    self.parent.mkdir(parents=True, exist_ok=True)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/pathlib.py", line 1323, in mkdir
+    self._accessor.mkdir(self, mode)
+OSError: [Errno 28] No space left on device: '/home/chunhui/.cache/huggingface/datasets/babylm_dataset_test/babylm_shuffle_deterministic21_10M_seed0'

wandb/run-20241105_160059-7almhy5u/files/requirements.txt

@@ -0,0 +1,147 @@
+funcsigs==1.0.2
+sentry-sdk==2.17.0
+multiprocess==0.70.16
+numpy==1.26.2
+pluralizer==1.2.0
+debugpy==1.6.7
+nvidia-cudnn-cu11==8.5.0.96
+deepspeed==0.15.2
+data==0.4
+pandas==2.1.3
+tomli==2.0.1
+charset-normalizer==3.3.2
+attrs==24.2.0
+aiosignal==1.3.1
+fsspec==2023.10.0
+nvidia-cusparse-cu11==11.7.4.91
+zipp==3.12.0
+mypy-extensions==1.0.0
+datasets==3.0.1
+joblib==1.3.2
+hjson==3.1.0
+traitlets==5.7.1
+stack-data==0.6.0
+transformers==4.45.1
+sympy==1.11.1
+Pygments==2.15.0
+docker-pycreds==0.4.0
+dill==0.3.8
+wheel==0.44.0
+prompt-toolkit==3.0.30
+parso==0.8.3
+ipykernel==6.23.1
+pyarrow==17.0.0
+certifi==2023.11.17
+nvidia-cufft-cu11==10.9.0.58
+six==1.16.0
+pydantic==2.9.2
+click==8.1.7
+nest-asyncio==1.5.6
+gmpy2==2.1.0
+matplotlib==3.8.2
+scipy==1.11.4
+typing_extensions==4.12.2
+statsmodels==0.14.0
+huggingface-hub==0.25.0
+frozenlist==1.4.1
+gpustat==1.1.1
+nvidia-nvtx-cu11==11.7.91
+safetensors==0.4.5
+stanza==1.9.2
+decorator==5.1.1
+seaborn==0.13.0
+sentencepiece==0.2.0
+PyYAML==6.0.1
+black==24.8.0
+protobuf==4.25.1
+pickleshare==0.7.5
+peft==0.13.0
+triton==2.0.0
+nvidia-cuda-runtime-cu11==11.7.99
+Jinja2==3.1.2
+nvidia-cusolver-cu11==11.4.0.1
+executing==1.2.0
+jupyter_client==8.1.0
+pluggy==1.3.0
+cmake==3.30.3
+pytz==2023.3.post1
+aiohappyeyeballs==2.4.2
+kiwisolver==1.4.5
+py-cpuinfo==9.0.0
+Pillow==10.1.0
+ptyprocess==0.7.0
+importlib_resources==6.4.5
+GitPython==3.1.43
+importlib-metadata==6.0.0
+iniconfig==2.0.0
+scikit-learn==1.3.2
+exceptiongroup==1.1.0
+networkx==2.8.6
+accelerate==1.0.0
+nltk==3.8.1
+shutilwhich==1.1.0
+fonttools==4.45.1
+future==0.18.3
+aiohttp==3.10.6
+wcwidth==0.2.5
+idna==3.6
+filelock==3.12.2
+pathspec==0.12.1
+jupyter_core==5.1.0
+lit==18.1.8
+nvidia-curand-cu11==10.2.10.91
+nvidia-cublas-cu11==11.10.3.66
+nvidia-ml-py==12.560.30
+msgpack==1.1.0
+python-dateutil==2.8.2
+blessed==1.20.0
+packaging==23.0
+gitdb==4.0.11
+yarl==1.13.0
+emoji==2.8.0
+tzdata==2023.3
+cycler==0.12.1
+tornado==6.2
+backcall==0.2.0
+plotnine==0.12.4
+ninja==1.11.1.1
+latex==0.7.0
+wandb==0.18.5
+setproctitle==1.3.3
+threadpoolctl==3.2.0
+requests==2.32.3
+pyparsing==3.1.1
+smmap==5.0.1
+pyzmq==23.0.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+matplotlib-inline==0.1.6
+latexcodec==1.0.0
+ipython==8.0.0
+patsy==0.5.3
+contourpy==1.2.0
+multidict==6.1.0
+mizani==0.9.3
+urllib3==2.1.0
+tokenizers==0.20.0
+MarkupSafe==2.1.2
+pip==24.2
+pexpect==4.8.0
+tqdm==4.66.5
+jedi==0.18.2
+pydantic_core==2.23.4
+tempdir==0.7.1
+mpmath==1.2.1
+setuptools==72.1.0
+pytest==7.4.3
+pure-eval==0.2.2
+psutil==5.9.1
+comm==0.1.2
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+regex==2023.10.3
+platformdirs==2.5.2
+asttokens==2.2.1
+torch==2.0.0
+nvidia-nccl-cu11==2.14.3
+xxhash==3.5.0

wandb/run-20241105_160059-7almhy5u/files/wandb-metadata.json

@@ -0,0 +1,44 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-11-05T21:00:59.303394Z",
+  "args":  [
+    "--perturbation",
+    "shuffle_deterministic21",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "3",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1792550322176"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  }
+}

wandb/run-20241105_160059-7almhy5u/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":5}}

wandb/run-20241105_160059-7almhy5u/logs/debug-internal.log

@@ -0,0 +1,17 @@
+{"time":"2024-11-05T16:00:59.306000637-05:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-11-05T16:00:59.306017137-05:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_160059-7almhy5u/logs/debug-core.log"}
+{"time":"2024-11-05T16:01:04.333616785-05:00","level":"INFO","msg":"created new stream","id":"7almhy5u"}
+{"time":"2024-11-05T16:01:04.333748536-05:00","level":"INFO","msg":"stream: started","id":"7almhy5u"}
+{"time":"2024-11-05T16:01:04.333787076-05:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"7almhy5u"}}
+{"time":"2024-11-05T16:01:04.333875716-05:00","level":"INFO","msg":"handler: started","stream_id":{"value":"7almhy5u"}}
+{"time":"2024-11-05T16:01:04.334271108-05:00","level":"INFO","msg":"sender: started","stream_id":"7almhy5u"}
+{"time":"2024-11-05T16:01:04.548552727-05:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-11-05T16:01:04.655616801-05:00","level":"INFO","msg":"stream: closing","id":"7almhy5u"}
+{"time":"2024-11-05T16:01:04.655650041-05:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-11-05T16:01:04.655684151-05:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-11-05T16:01:04.922819935-05:00","level":"ERROR","msg":"sender: sendDefer: failed to build job artifact","error":"failed to write data to file: write /tmp/tmpfile-2450422308: no space left on device"}
+{"time":"2024-11-05T16:01:05.17053313-05:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-11-05T16:01:05.515906204-05:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"7almhy5u"}}
+{"time":"2024-11-05T16:01:05.515951064-05:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"7almhy5u"}}
+{"time":"2024-11-05T16:01:05.515969004-05:00","level":"INFO","msg":"sender: closed","stream_id":"7almhy5u"}
+{"time":"2024-11-05T16:01:05.516029494-05:00","level":"INFO","msg":"stream: closed","id":"7almhy5u"}

wandb/run-20241105_160059-7almhy5u/logs/debug.log

@@ -0,0 +1,27 @@
+2024-11-05 16:00:59,301 INFO    MainThread:1769881 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-11-05 16:00:59,301 INFO    MainThread:1769881 [wandb_setup.py:_flush():79] Configure stats pid to 1769881
+2024-11-05 16:00:59,301 INFO    MainThread:1769881 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-11-05 16:00:59,301 INFO    MainThread:1769881 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-11-05 16:00:59,301 INFO    MainThread:1769881 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-11-05 16:00:59,301 INFO    MainThread:1769881 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-11-05 16:00:59,301 INFO    MainThread:1769881 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-11-05 16:00:59,301 INFO    MainThread:1769881 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-11-05 16:00:59,301 INFO    MainThread:1769881 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_160059-7almhy5u/logs/debug.log
+2024-11-05 16:00:59,301 INFO    MainThread:1769881 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_160059-7almhy5u/logs/debug-internal.log
+2024-11-05 16:00:59,301 INFO    MainThread:1769881 [wandb_init.py:init():621] calling init triggers
+2024-11-05 16:00:59,301 INFO    MainThread:1769881 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-11-05 16:00:59,301 INFO    MainThread:1769881 [wandb_init.py:init():671] starting backend
+2024-11-05 16:00:59,301 INFO    MainThread:1769881 [wandb_init.py:init():675] sending inform_init request
+2024-11-05 16:00:59,302 INFO    MainThread:1769881 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-11-05 16:00:59,303 INFO    MainThread:1769881 [wandb_init.py:init():688] backend started and connected
+2024-11-05 16:00:59,306 INFO    MainThread:1769881 [wandb_init.py:init():783] updated telemetry
+2024-11-05 16:00:59,334 INFO    MainThread:1769881 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-11-05 16:01:04,545 INFO    MainThread:1769881 [wandb_init.py:init():867] starting run threads in backend
+2024-11-05 16:01:04,631 INFO    MainThread:1769881 [wandb_run.py:_console_start():2463] atexit reg
+2024-11-05 16:01:04,631 INFO    MainThread:1769881 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-11-05 16:01:04,632 INFO    MainThread:1769881 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-11-05 16:01:04,632 INFO    MainThread:1769881 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-11-05 16:01:04,633 INFO    MainThread:1769881 [wandb_init.py:init():911] run started, returning control to user process
+2024-11-05 16:01:04,633 INFO    MainThread:1769881 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_deterministic21', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0, 'lr': 5e-06}
+2024-11-05 16:01:04,655 WARNING MsgRouterThr:1769881 [router.py:message_loop():77] message_loop has been closed