Add files using upload-large-folder tool

.gitattributes

@@ -124,3 +124,10 @@ wandb/run-20241107_160909-8q592a50/run-8q592a50.wandb filter=lfs diff=lfs merge=
 wandb/run-20241130_000210-gk41m5w8/run-gk41m5w8.wandb filter=lfs diff=lfs merge=lfs -text
 wandb/run-20241115_125218-bbgu27nw/run-bbgu27nw.wandb filter=lfs diff=lfs merge=lfs -text
 wandb/run-20241030_013340-zxn5usuc/run-zxn5usuc.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241030_013340-9mu0k2by/run-9mu0k2by.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241031_002020-ifaaozhq/run-ifaaozhq.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241030_233740-anh3ext7/run-anh3ext7.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241105_163244-o1vw2gev/run-o1vw2gev.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241115_125218-wtoze6lw/run-wtoze6lw.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241030_112852-mfvd6tgw/run-mfvd6tgw.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241031_114700-78zg7gu4/run-78zg7gu4.wandb filter=lfs diff=lfs merge=lfs -text

wandb/run-20241030_010759-qbmaa3bw/files/config.yaml

@@ -0,0 +1,47 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.9.19
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "3":
+                - 13
+                - 23
+                - 55
+            "4": 3.9.19
+            "5": 0.18.5
+            "6": 4.45.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+batch_size:
+    value: 3
+epoch:
+    value: 7
+perturbation:
+    value: reverse_control
+seed:
+    value: 0
+train_set:
+    value: 10M

wandb/run-20241030_010759-qbmaa3bw/files/output.log

@@ -0,0 +1,4 @@
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 162, in <module>
+    dataset_name = f"babylm_{args.perturbation}_{args.train_zset}_seed{args.seed}"
+AttributeError: 'Namespace' object has no attribute 'train_zset'

wandb/run-20241030_010759-qbmaa3bw/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-10-30T05:07:59.162763Z",
+  "args":  [
+    "--perturbation",
+    "reverse_control",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "7",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1719200272384"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241030_010759-qbmaa3bw/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":0}}

wandb/run-20241030_010759-qbmaa3bw/logs/debug-internal.log

@@ -0,0 +1,16 @@
+{"time":"2024-10-30T01:07:59.16436947-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-10-30T01:07:59.16437974-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_010759-qbmaa3bw/logs/debug-core.log"}
+{"time":"2024-10-30T01:07:59.27003612-04:00","level":"INFO","msg":"created new stream","id":"qbmaa3bw"}
+{"time":"2024-10-30T01:07:59.27006683-04:00","level":"INFO","msg":"stream: started","id":"qbmaa3bw"}
+{"time":"2024-10-30T01:07:59.2701315-04:00","level":"INFO","msg":"sender: started","stream_id":"qbmaa3bw"}
+{"time":"2024-10-30T01:07:59.27010472-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"qbmaa3bw"}}
+{"time":"2024-10-30T01:07:59.27013166-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"qbmaa3bw"}}
+{"time":"2024-10-30T01:07:59.514334253-04:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-10-30T01:07:59.614240043-04:00","level":"INFO","msg":"stream: closing","id":"qbmaa3bw"}
+{"time":"2024-10-30T01:07:59.614272313-04:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-10-30T01:07:59.614660656-04:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-10-30T01:08:00.142100917-04:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-10-30T01:08:00.254410602-04:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"qbmaa3bw"}}
+{"time":"2024-10-30T01:08:00.254433072-04:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"qbmaa3bw"}}
+{"time":"2024-10-30T01:08:00.254451802-04:00","level":"INFO","msg":"sender: closed","stream_id":"qbmaa3bw"}
+{"time":"2024-10-30T01:08:00.254475442-04:00","level":"INFO","msg":"stream: closed","id":"qbmaa3bw"}

wandb/run-20241030_010759-qbmaa3bw/logs/debug.log

@@ -0,0 +1,27 @@
+2024-10-30 01:07:59,160 INFO    MainThread:322463 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-10-30 01:07:59,161 INFO    MainThread:322463 [wandb_setup.py:_flush():79] Configure stats pid to 322463
+2024-10-30 01:07:59,161 INFO    MainThread:322463 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-10-30 01:07:59,161 INFO    MainThread:322463 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-10-30 01:07:59,161 INFO    MainThread:322463 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-30 01:07:59,161 INFO    MainThread:322463 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-30 01:07:59,161 INFO    MainThread:322463 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-10-30 01:07:59,161 INFO    MainThread:322463 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-30 01:07:59,161 INFO    MainThread:322463 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_010759-qbmaa3bw/logs/debug.log
+2024-10-30 01:07:59,161 INFO    MainThread:322463 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_010759-qbmaa3bw/logs/debug-internal.log
+2024-10-30 01:07:59,161 INFO    MainThread:322463 [wandb_init.py:init():621] calling init triggers
+2024-10-30 01:07:59,161 INFO    MainThread:322463 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-10-30 01:07:59,161 INFO    MainThread:322463 [wandb_init.py:init():671] starting backend
+2024-10-30 01:07:59,161 INFO    MainThread:322463 [wandb_init.py:init():675] sending inform_init request
+2024-10-30 01:07:59,162 INFO    MainThread:322463 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-30 01:07:59,162 INFO    MainThread:322463 [wandb_init.py:init():688] backend started and connected
+2024-10-30 01:07:59,165 INFO    MainThread:322463 [wandb_init.py:init():783] updated telemetry
+2024-10-30 01:07:59,200 INFO    MainThread:322463 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-10-30 01:07:59,511 INFO    MainThread:322463 [wandb_init.py:init():867] starting run threads in backend
+2024-10-30 01:07:59,610 INFO    MainThread:322463 [wandb_run.py:_console_start():2463] atexit reg
+2024-10-30 01:07:59,611 INFO    MainThread:322463 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-10-30 01:07:59,611 INFO    MainThread:322463 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-10-30 01:07:59,611 INFO    MainThread:322463 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-10-30 01:07:59,613 INFO    MainThread:322463 [wandb_init.py:init():911] run started, returning control to user process
+2024-10-30 01:07:59,613 INFO    MainThread:322463 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_control', 'train_set': '10M', 'batch_size': 3, 'epoch': 7, 'seed': 0}
+2024-10-30 01:07:59,614 WARNING MsgRouterThr:322463 [router.py:message_loop():77] message_loop has been closed

wandb/run-20241030_012617-b4q8kp96/files/output.log

@@ -0,0 +1,2 @@
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:04<00:00,  2.34s/it]
+Map:  11%|██████████████                                                                                                                  | 2000/18140 [00:06<00:52, 307.63 examples/s]

wandb/run-20241030_012617-b4q8kp96/files/requirements.txt

@@ -0,0 +1,147 @@
+funcsigs==1.0.2
+sentry-sdk==2.17.0
+multiprocess==0.70.16
+numpy==1.26.2
+pluralizer==1.2.0
+debugpy==1.6.7
+nvidia-cudnn-cu11==8.5.0.96
+deepspeed==0.15.2
+data==0.4
+pandas==2.1.3
+tomli==2.0.1
+charset-normalizer==3.3.2
+attrs==24.2.0
+aiosignal==1.3.1
+fsspec==2023.10.0
+nvidia-cusparse-cu11==11.7.4.91
+zipp==3.12.0
+mypy-extensions==1.0.0
+datasets==3.0.1
+joblib==1.3.2
+hjson==3.1.0
+traitlets==5.7.1
+stack-data==0.6.0
+transformers==4.45.1
+sympy==1.11.1
+Pygments==2.15.0
+docker-pycreds==0.4.0
+dill==0.3.8
+wheel==0.44.0
+prompt-toolkit==3.0.30
+parso==0.8.3
+ipykernel==6.23.1
+pyarrow==17.0.0
+certifi==2023.11.17
+nvidia-cufft-cu11==10.9.0.58
+six==1.16.0
+pydantic==2.9.2
+click==8.1.7
+nest-asyncio==1.5.6
+gmpy2==2.1.0
+matplotlib==3.8.2
+scipy==1.11.4
+typing_extensions==4.12.2
+statsmodels==0.14.0
+huggingface-hub==0.25.0
+frozenlist==1.4.1
+gpustat==1.1.1
+nvidia-nvtx-cu11==11.7.91
+safetensors==0.4.5
+stanza==1.9.2
+decorator==5.1.1
+seaborn==0.13.0
+sentencepiece==0.2.0
+PyYAML==6.0.1
+black==24.8.0
+protobuf==4.25.1
+pickleshare==0.7.5
+peft==0.13.0
+triton==2.0.0
+nvidia-cuda-runtime-cu11==11.7.99
+Jinja2==3.1.2
+nvidia-cusolver-cu11==11.4.0.1
+executing==1.2.0
+jupyter_client==8.1.0
+pluggy==1.3.0
+cmake==3.30.3
+pytz==2023.3.post1
+aiohappyeyeballs==2.4.2
+kiwisolver==1.4.5
+py-cpuinfo==9.0.0
+Pillow==10.1.0
+ptyprocess==0.7.0
+importlib_resources==6.4.5
+GitPython==3.1.43
+importlib-metadata==6.0.0
+iniconfig==2.0.0
+scikit-learn==1.3.2
+exceptiongroup==1.1.0
+networkx==2.8.6
+accelerate==1.0.0
+nltk==3.8.1
+shutilwhich==1.1.0
+fonttools==4.45.1
+future==0.18.3
+aiohttp==3.10.6
+wcwidth==0.2.5
+idna==3.6
+filelock==3.12.2
+pathspec==0.12.1
+jupyter_core==5.1.0
+lit==18.1.8
+nvidia-curand-cu11==10.2.10.91
+nvidia-cublas-cu11==11.10.3.66
+nvidia-ml-py==12.560.30
+msgpack==1.1.0
+python-dateutil==2.8.2
+blessed==1.20.0
+packaging==23.0
+gitdb==4.0.11
+yarl==1.13.0
+emoji==2.8.0
+tzdata==2023.3
+cycler==0.12.1
+tornado==6.2
+backcall==0.2.0
+plotnine==0.12.4
+ninja==1.11.1.1
+latex==0.7.0
+wandb==0.18.5
+setproctitle==1.3.3
+threadpoolctl==3.2.0
+requests==2.32.3
+pyparsing==3.1.1
+smmap==5.0.1
+pyzmq==23.0.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+matplotlib-inline==0.1.6
+latexcodec==1.0.0
+ipython==8.0.0
+patsy==0.5.3
+contourpy==1.2.0
+multidict==6.1.0
+mizani==0.9.3
+urllib3==2.1.0
+tokenizers==0.20.0
+MarkupSafe==2.1.2
+pip==24.2
+pexpect==4.8.0
+tqdm==4.66.5
+jedi==0.18.2
+pydantic_core==2.23.4
+tempdir==0.7.1
+mpmath==1.2.1
+setuptools==72.1.0
+pytest==7.4.3
+pure-eval==0.2.2
+psutil==5.9.1
+comm==0.1.2
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+regex==2023.10.3
+platformdirs==2.5.2
+asttokens==2.2.1
+torch==2.0.0
+nvidia-nccl-cu11==2.14.3
+xxhash==3.5.0

wandb/run-20241030_012617-b4q8kp96/logs/debug-internal.log

@@ -0,0 +1,8 @@
+{"time":"2024-10-30T01:26:17.477621389-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-10-30T01:26:17.477638659-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_012617-b4q8kp96/logs/debug-core.log"}
+{"time":"2024-10-30T01:26:17.591821467-04:00","level":"INFO","msg":"created new stream","id":"b4q8kp96"}
+{"time":"2024-10-30T01:26:17.591888718-04:00","level":"INFO","msg":"stream: started","id":"b4q8kp96"}
+{"time":"2024-10-30T01:26:17.591930648-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"b4q8kp96"}}
+{"time":"2024-10-30T01:26:17.59223234-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"b4q8kp96"}}
+{"time":"2024-10-30T01:26:17.59225328-04:00","level":"INFO","msg":"sender: started","stream_id":"b4q8kp96"}
+{"time":"2024-10-30T01:26:17.898671113-04:00","level":"INFO","msg":"Starting system monitor"}

wandb/run-20241030_012617-b4q8kp96/logs/debug.log

@@ -0,0 +1,26 @@
+2024-10-30 01:26:17,470 INFO    MainThread:332625 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-10-30 01:26:17,471 INFO    MainThread:332625 [wandb_setup.py:_flush():79] Configure stats pid to 332625
+2024-10-30 01:26:17,471 INFO    MainThread:332625 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-10-30 01:26:17,471 INFO    MainThread:332625 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-10-30 01:26:17,471 INFO    MainThread:332625 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-30 01:26:17,471 INFO    MainThread:332625 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-30 01:26:17,471 INFO    MainThread:332625 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-10-30 01:26:17,471 INFO    MainThread:332625 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-30 01:26:17,471 INFO    MainThread:332625 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_012617-b4q8kp96/logs/debug.log
+2024-10-30 01:26:17,471 INFO    MainThread:332625 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_012617-b4q8kp96/logs/debug-internal.log
+2024-10-30 01:26:17,471 INFO    MainThread:332625 [wandb_init.py:init():621] calling init triggers
+2024-10-30 01:26:17,471 INFO    MainThread:332625 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-10-30 01:26:17,471 INFO    MainThread:332625 [wandb_init.py:init():671] starting backend
+2024-10-30 01:26:17,471 INFO    MainThread:332625 [wandb_init.py:init():675] sending inform_init request
+2024-10-30 01:26:17,476 INFO    MainThread:332625 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-30 01:26:17,477 INFO    MainThread:332625 [wandb_init.py:init():688] backend started and connected
+2024-10-30 01:26:17,481 INFO    MainThread:332625 [wandb_init.py:init():783] updated telemetry
+2024-10-30 01:26:17,540 INFO    MainThread:332625 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-10-30 01:26:17,890 INFO    MainThread:332625 [wandb_init.py:init():867] starting run threads in backend
+2024-10-30 01:26:18,025 INFO    MainThread:332625 [wandb_run.py:_console_start():2463] atexit reg
+2024-10-30 01:26:18,025 INFO    MainThread:332625 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-10-30 01:26:18,025 INFO    MainThread:332625 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-10-30 01:26:18,025 INFO    MainThread:332625 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-10-30 01:26:18,027 INFO    MainThread:332625 [wandb_init.py:init():911] run started, returning control to user process
+2024-10-30 01:26:18,027 INFO    MainThread:332625 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_control', 'train_set': '10M', 'batch_size': 3, 'epoch': 7, 'seed': 0}

wandb/run-20241030_013141-i752ozdz/files/config.yaml

@@ -0,0 +1,47 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.9.19
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "3":
+                - 13
+                - 23
+                - 55
+            "4": 3.9.19
+            "5": 0.18.5
+            "6": 4.45.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+batch_size:
+    value: 3
+epoch:
+    value: 7
+perturbation:
+    value: reverse_full
+seed:
+    value: 0
+train_set:
+    value: 10M

wandb/run-20241030_013141-i752ozdz/files/output.log

@@ -0,0 +1,42 @@
+Downloading shards:   0%|                                                                                                                                        | 0/2 [01:04<?, ?it/s]
+Error in sys.excepthook:
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/wandb/sdk/lib/exit_hooks.py", line 52, in exc_handler
+    traceback.print_exception(exc_type, exc, tb)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/traceback.py", line 103, in print_exception
+    for line in TracebackException(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/traceback.py", line 517, in __init__
+    self.stack = StackSummary.extract(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/traceback.py", line 353, in extract
+    linecache.lazycache(filename, f.f_globals)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/linecache.py", line 170, in lazycache
+    loader = module_globals['__loader__']
+KeyboardInterrupt
+
+Original exception was:
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 172, in <module>
+    model = AutoModelForCausalLM.from_pretrained(model_name,
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/models/auto/auto_factory.py", line 564, in from_pretrained
+    return model_class.from_pretrained(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/modeling_utils.py", line 3769, in from_pretrained
+    resolved_archive_file, sharded_metadata = get_checkpoint_shard_files(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/utils/hub.py", line 1098, in get_checkpoint_shard_files
+    cached_filename = cached_file(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/utils/hub.py", line 403, in cached_file
+    resolved_file = hf_hub_download(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/utils/_deprecation.py", line 101, in inner_f
+    return f(*args, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn
+    return fn(*args, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1232, in hf_hub_download
+    return _hf_hub_download_to_cache_dir(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1380, in _hf_hub_download_to_cache_dir
+    with WeakFileLock(lock_path):
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/contextlib.py", line 119, in __enter__
+    return next(self.gen)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/utils/_fixes.py", line 98, in WeakFileLock
+    lock.acquire()
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/filelock/_api.py", line 225, in acquire
+    time.sleep(poll_interval)
+KeyboardInterrupt

wandb/run-20241030_013141-i752ozdz/files/requirements.txt

@@ -0,0 +1,147 @@
+funcsigs==1.0.2
+sentry-sdk==2.17.0
+multiprocess==0.70.16
+numpy==1.26.2
+pluralizer==1.2.0
+debugpy==1.6.7
+nvidia-cudnn-cu11==8.5.0.96
+deepspeed==0.15.2
+data==0.4
+pandas==2.1.3
+tomli==2.0.1
+charset-normalizer==3.3.2
+attrs==24.2.0
+aiosignal==1.3.1
+fsspec==2023.10.0
+nvidia-cusparse-cu11==11.7.4.91
+zipp==3.12.0
+mypy-extensions==1.0.0
+datasets==3.0.1
+joblib==1.3.2
+hjson==3.1.0
+traitlets==5.7.1
+stack-data==0.6.0
+transformers==4.45.1
+sympy==1.11.1
+Pygments==2.15.0
+docker-pycreds==0.4.0
+dill==0.3.8
+wheel==0.44.0
+prompt-toolkit==3.0.30
+parso==0.8.3
+ipykernel==6.23.1
+pyarrow==17.0.0
+certifi==2023.11.17
+nvidia-cufft-cu11==10.9.0.58
+six==1.16.0
+pydantic==2.9.2
+click==8.1.7
+nest-asyncio==1.5.6
+gmpy2==2.1.0
+matplotlib==3.8.2
+scipy==1.11.4
+typing_extensions==4.12.2
+statsmodels==0.14.0
+huggingface-hub==0.25.0
+frozenlist==1.4.1
+gpustat==1.1.1
+nvidia-nvtx-cu11==11.7.91
+safetensors==0.4.5
+stanza==1.9.2
+decorator==5.1.1
+seaborn==0.13.0
+sentencepiece==0.2.0
+PyYAML==6.0.1
+black==24.8.0
+protobuf==4.25.1
+pickleshare==0.7.5
+peft==0.13.0
+triton==2.0.0
+nvidia-cuda-runtime-cu11==11.7.99
+Jinja2==3.1.2
+nvidia-cusolver-cu11==11.4.0.1
+executing==1.2.0
+jupyter_client==8.1.0
+pluggy==1.3.0
+cmake==3.30.3
+pytz==2023.3.post1
+aiohappyeyeballs==2.4.2
+kiwisolver==1.4.5
+py-cpuinfo==9.0.0
+Pillow==10.1.0
+ptyprocess==0.7.0
+importlib_resources==6.4.5
+GitPython==3.1.43
+importlib-metadata==6.0.0
+iniconfig==2.0.0
+scikit-learn==1.3.2
+exceptiongroup==1.1.0
+networkx==2.8.6
+accelerate==1.0.0
+nltk==3.8.1
+shutilwhich==1.1.0
+fonttools==4.45.1
+future==0.18.3
+aiohttp==3.10.6
+wcwidth==0.2.5
+idna==3.6
+filelock==3.12.2
+pathspec==0.12.1
+jupyter_core==5.1.0
+lit==18.1.8
+nvidia-curand-cu11==10.2.10.91
+nvidia-cublas-cu11==11.10.3.66
+nvidia-ml-py==12.560.30
+msgpack==1.1.0
+python-dateutil==2.8.2
+blessed==1.20.0
+packaging==23.0
+gitdb==4.0.11
+yarl==1.13.0
+emoji==2.8.0
+tzdata==2023.3
+cycler==0.12.1
+tornado==6.2
+backcall==0.2.0
+plotnine==0.12.4
+ninja==1.11.1.1
+latex==0.7.0
+wandb==0.18.5
+setproctitle==1.3.3
+threadpoolctl==3.2.0
+requests==2.32.3
+pyparsing==3.1.1
+smmap==5.0.1
+pyzmq==23.0.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+matplotlib-inline==0.1.6
+latexcodec==1.0.0
+ipython==8.0.0
+patsy==0.5.3
+contourpy==1.2.0
+multidict==6.1.0
+mizani==0.9.3
+urllib3==2.1.0
+tokenizers==0.20.0
+MarkupSafe==2.1.2
+pip==24.2
+pexpect==4.8.0
+tqdm==4.66.5
+jedi==0.18.2
+pydantic_core==2.23.4
+tempdir==0.7.1
+mpmath==1.2.1
+setuptools==72.1.0
+pytest==7.4.3
+pure-eval==0.2.2
+psutil==5.9.1
+comm==0.1.2
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+regex==2023.10.3
+platformdirs==2.5.2
+asttokens==2.2.1
+torch==2.0.0
+nvidia-nccl-cu11==2.14.3
+xxhash==3.5.0

wandb/run-20241030_013141-i752ozdz/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-10-30T05:31:41.732724Z",
+  "args":  [
+    "--perturbation",
+    "reverse_full",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "7",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1709824413696"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241030_013141-i752ozdz/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":94}}

wandb/run-20241030_013141-i752ozdz/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2024-10-30T01:31:41.735329435-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-10-30T01:31:41.735346675-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_013141-i752ozdz/logs/debug-core.log"}
+{"time":"2024-10-30T01:31:41.841916432-04:00","level":"INFO","msg":"created new stream","id":"i752ozdz"}
+{"time":"2024-10-30T01:31:41.841966143-04:00","level":"INFO","msg":"stream: started","id":"i752ozdz"}
+{"time":"2024-10-30T01:31:41.841968243-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"i752ozdz"}}
+{"time":"2024-10-30T01:31:41.842006353-04:00","level":"INFO","msg":"sender: started","stream_id":"i752ozdz"}
+{"time":"2024-10-30T01:31:41.842000233-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"i752ozdz"}}
+{"time":"2024-10-30T01:31:42.05562675-04:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-10-30T01:33:16.596067109-04:00","level":"INFO","msg":"stream: closing","id":"i752ozdz"}
+{"time":"2024-10-30T01:33:16.59615142-04:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-10-30T01:33:16.597372638-04:00","level":"INFO","msg":"Stopped system monitor"}

wandb/run-20241030_013141-i752ozdz/logs/debug.log

@@ -0,0 +1,27 @@
+2024-10-30 01:31:41,730 INFO    MainThread:335757 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-10-30 01:31:41,730 INFO    MainThread:335757 [wandb_setup.py:_flush():79] Configure stats pid to 335757
+2024-10-30 01:31:41,730 INFO    MainThread:335757 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-10-30 01:31:41,730 INFO    MainThread:335757 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-10-30 01:31:41,730 INFO    MainThread:335757 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-30 01:31:41,730 INFO    MainThread:335757 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-30 01:31:41,730 INFO    MainThread:335757 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-10-30 01:31:41,730 INFO    MainThread:335757 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-30 01:31:41,730 INFO    MainThread:335757 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_013141-i752ozdz/logs/debug.log
+2024-10-30 01:31:41,730 INFO    MainThread:335757 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_013141-i752ozdz/logs/debug-internal.log
+2024-10-30 01:31:41,730 INFO    MainThread:335757 [wandb_init.py:init():621] calling init triggers
+2024-10-30 01:31:41,730 INFO    MainThread:335757 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-10-30 01:31:41,730 INFO    MainThread:335757 [wandb_init.py:init():671] starting backend
+2024-10-30 01:31:41,731 INFO    MainThread:335757 [wandb_init.py:init():675] sending inform_init request
+2024-10-30 01:31:41,732 INFO    MainThread:335757 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-30 01:31:41,732 INFO    MainThread:335757 [wandb_init.py:init():688] backend started and connected
+2024-10-30 01:31:41,735 INFO    MainThread:335757 [wandb_init.py:init():783] updated telemetry
+2024-10-30 01:31:41,780 INFO    MainThread:335757 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-10-30 01:31:42,052 INFO    MainThread:335757 [wandb_init.py:init():867] starting run threads in backend
+2024-10-30 01:31:42,160 INFO    MainThread:335757 [wandb_run.py:_console_start():2463] atexit reg
+2024-10-30 01:31:42,160 INFO    MainThread:335757 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-10-30 01:31:42,160 INFO    MainThread:335757 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-10-30 01:31:42,160 INFO    MainThread:335757 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-10-30 01:31:42,162 INFO    MainThread:335757 [wandb_init.py:init():911] run started, returning control to user process
+2024-10-30 01:31:42,162 INFO    MainThread:335757 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_full', 'train_set': '10M', 'batch_size': 3, 'epoch': 7, 'seed': 0}
+2024-10-30 01:33:16,596 WARNING MsgRouterThr:335757 [router.py:message_loop():77] message_loop has been closed

wandb/run-20241030_013340-9mu0k2by/run-9mu0k2by.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0454390c188daab06695ab4133c8d035c058ff54eff87a02febfbe6e081e0439
+size 20119552

wandb/run-20241030_112852-mfvd6tgw/run-mfvd6tgw.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb7ec612387d4e6fc9c7860249a29a996dbd177d1a16fe0b117580843420bf33
+size 14117946

wandb/run-20241030_222932-48sfxfqm/files/output.log

@@ -0,0 +1,14 @@
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:18<00:00,  9.39s/it]
+tokenized_valid: Dataset({
+    features: ['input_ids', 'attention_mask'],
+    num_rows: 600
+})
+/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
+  warnings.warn(
+[2024-10-30 22:29:54,088] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2024-10-30 22:30:03,533] [INFO] [comm.py:652:init_distributed] cdb=None
+Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination
+Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...
+Loading extension module cpu_adam...
+Time to load cpu_adam op: 5.216655731201172 seconds
+wandb: WARNING Fatal error while uploading data. Some run data will not be synced, but it will still be written to disk. Use `wandb sync` at the end of the run to try uploading.

wandb/run-20241030_222932-48sfxfqm/files/requirements.txt

@@ -0,0 +1,147 @@
+funcsigs==1.0.2
+sentry-sdk==2.17.0
+multiprocess==0.70.16
+numpy==1.26.2
+pluralizer==1.2.0
+debugpy==1.6.7
+nvidia-cudnn-cu11==8.5.0.96
+deepspeed==0.15.2
+data==0.4
+pandas==2.1.3
+tomli==2.0.1
+charset-normalizer==3.3.2
+attrs==24.2.0
+aiosignal==1.3.1
+fsspec==2023.10.0
+nvidia-cusparse-cu11==11.7.4.91
+zipp==3.12.0
+mypy-extensions==1.0.0
+datasets==3.0.1
+joblib==1.3.2
+hjson==3.1.0
+traitlets==5.7.1
+stack-data==0.6.0
+transformers==4.45.1
+sympy==1.11.1
+Pygments==2.15.0
+docker-pycreds==0.4.0
+dill==0.3.8
+wheel==0.44.0
+prompt-toolkit==3.0.30
+parso==0.8.3
+ipykernel==6.23.1
+pyarrow==17.0.0
+certifi==2023.11.17
+nvidia-cufft-cu11==10.9.0.58
+six==1.16.0
+pydantic==2.9.2
+click==8.1.7
+nest-asyncio==1.5.6
+gmpy2==2.1.0
+matplotlib==3.8.2
+scipy==1.11.4
+typing_extensions==4.12.2
+statsmodels==0.14.0
+huggingface-hub==0.25.0
+frozenlist==1.4.1
+gpustat==1.1.1
+nvidia-nvtx-cu11==11.7.91
+safetensors==0.4.5
+stanza==1.9.2
+decorator==5.1.1
+seaborn==0.13.0
+sentencepiece==0.2.0
+PyYAML==6.0.1
+black==24.8.0
+protobuf==4.25.1
+pickleshare==0.7.5
+peft==0.13.0
+triton==2.0.0
+nvidia-cuda-runtime-cu11==11.7.99
+Jinja2==3.1.2
+nvidia-cusolver-cu11==11.4.0.1
+executing==1.2.0
+jupyter_client==8.1.0
+pluggy==1.3.0
+cmake==3.30.3
+pytz==2023.3.post1
+aiohappyeyeballs==2.4.2
+kiwisolver==1.4.5
+py-cpuinfo==9.0.0
+Pillow==10.1.0
+ptyprocess==0.7.0
+importlib_resources==6.4.5
+GitPython==3.1.43
+importlib-metadata==6.0.0
+iniconfig==2.0.0
+scikit-learn==1.3.2
+exceptiongroup==1.1.0
+networkx==2.8.6
+accelerate==1.0.0
+nltk==3.8.1
+shutilwhich==1.1.0
+fonttools==4.45.1
+future==0.18.3
+aiohttp==3.10.6
+wcwidth==0.2.5
+idna==3.6
+filelock==3.12.2
+pathspec==0.12.1
+jupyter_core==5.1.0
+lit==18.1.8
+nvidia-curand-cu11==10.2.10.91
+nvidia-cublas-cu11==11.10.3.66
+nvidia-ml-py==12.560.30
+msgpack==1.1.0
+python-dateutil==2.8.2
+blessed==1.20.0
+packaging==23.0
+gitdb==4.0.11
+yarl==1.13.0
+emoji==2.8.0
+tzdata==2023.3
+cycler==0.12.1
+tornado==6.2
+backcall==0.2.0
+plotnine==0.12.4
+ninja==1.11.1.1
+latex==0.7.0
+wandb==0.18.5
+setproctitle==1.3.3
+threadpoolctl==3.2.0
+requests==2.32.3
+pyparsing==3.1.1
+smmap==5.0.1
+pyzmq==23.0.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+matplotlib-inline==0.1.6
+latexcodec==1.0.0
+ipython==8.0.0
+patsy==0.5.3
+contourpy==1.2.0
+multidict==6.1.0
+mizani==0.9.3
+urllib3==2.1.0
+tokenizers==0.20.0
+MarkupSafe==2.1.2
+pip==24.2
+pexpect==4.8.0
+tqdm==4.66.5
+jedi==0.18.2
+pydantic_core==2.23.4
+tempdir==0.7.1
+mpmath==1.2.1
+setuptools==72.1.0
+pytest==7.4.3
+pure-eval==0.2.2
+psutil==5.9.1
+comm==0.1.2
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+regex==2023.10.3
+platformdirs==2.5.2
+asttokens==2.2.1
+torch==2.0.0
+nvidia-nccl-cu11==2.14.3
+xxhash==3.5.0

wandb/run-20241030_222932-48sfxfqm/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-10-31T02:29:32.385161Z",
+  "args":  [
+    "--perturbation",
+    "reverse_control",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "3",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1710969503744"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241030_222932-48sfxfqm/logs/debug-internal.log

@@ -0,0 +1,10 @@
+{"time":"2024-10-30T22:29:32.387733734-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-10-30T22:29:32.387749274-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_222932-48sfxfqm/logs/debug-core.log"}
+{"time":"2024-10-30T22:29:32.494517278-04:00","level":"INFO","msg":"created new stream","id":"48sfxfqm"}
+{"time":"2024-10-30T22:29:32.494545718-04:00","level":"INFO","msg":"stream: started","id":"48sfxfqm"}
+{"time":"2024-10-30T22:29:32.494641869-04:00","level":"INFO","msg":"sender: started","stream_id":"48sfxfqm"}
+{"time":"2024-10-30T22:29:32.494581959-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"48sfxfqm"}}
+{"time":"2024-10-30T22:29:32.494579609-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"48sfxfqm"}}
+{"time":"2024-10-30T22:29:32.696001306-04:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-10-30T22:55:03.122546482-04:00","level":"ERROR","msg":"HTTP error","status":404,"method":"POST","url":"https://api.wandb.ai/files/yaning1001-dartmouth-college/impossible_llm_reverse/48sfxfqm/file_stream"}
+{"time":"2024-10-30T22:55:03.127159548-04:00","level":"ERROR+4","msg":"filestream: fatal error: filestream: failed to upload: 404 Not Found path=files/yaning1001-dartmouth-college/impossible_llm_reverse/48sfxfqm/file_stream: {\"error\":\"run impossible_llm_reverse/48sfxfqm not found while streaming file\"}"}

wandb/run-20241030_222932-48sfxfqm/logs/debug.log

@@ -0,0 +1,26 @@
+2024-10-30 22:29:32,383 INFO    MainThread:447697 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-10-30 22:29:32,383 INFO    MainThread:447697 [wandb_setup.py:_flush():79] Configure stats pid to 447697
+2024-10-30 22:29:32,383 INFO    MainThread:447697 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-10-30 22:29:32,383 INFO    MainThread:447697 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-10-30 22:29:32,383 INFO    MainThread:447697 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-30 22:29:32,383 INFO    MainThread:447697 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-30 22:29:32,383 INFO    MainThread:447697 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-10-30 22:29:32,383 INFO    MainThread:447697 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-30 22:29:32,383 INFO    MainThread:447697 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_222932-48sfxfqm/logs/debug.log
+2024-10-30 22:29:32,383 INFO    MainThread:447697 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_222932-48sfxfqm/logs/debug-internal.log
+2024-10-30 22:29:32,383 INFO    MainThread:447697 [wandb_init.py:init():621] calling init triggers
+2024-10-30 22:29:32,383 INFO    MainThread:447697 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-10-30 22:29:32,383 INFO    MainThread:447697 [wandb_init.py:init():671] starting backend
+2024-10-30 22:29:32,383 INFO    MainThread:447697 [wandb_init.py:init():675] sending inform_init request
+2024-10-30 22:29:32,384 INFO    MainThread:447697 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-30 22:29:32,384 INFO    MainThread:447697 [wandb_init.py:init():688] backend started and connected
+2024-10-30 22:29:32,387 INFO    MainThread:447697 [wandb_init.py:init():783] updated telemetry
+2024-10-30 22:29:32,432 INFO    MainThread:447697 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-10-30 22:29:32,693 INFO    MainThread:447697 [wandb_init.py:init():867] starting run threads in backend
+2024-10-30 22:29:32,840 INFO    MainThread:447697 [wandb_run.py:_console_start():2463] atexit reg
+2024-10-30 22:29:32,840 INFO    MainThread:447697 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-10-30 22:29:32,840 INFO    MainThread:447697 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-10-30 22:29:32,840 INFO    MainThread:447697 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-10-30 22:29:32,842 INFO    MainThread:447697 [wandb_init.py:init():911] run started, returning control to user process
+2024-10-30 22:29:32,843 INFO    MainThread:447697 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_control', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0}

wandb/run-20241030_222932-9ywqkbmn/files/output.log

@@ -0,0 +1,40 @@
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:18<00:00,  9.39s/it]
+tokenized_valid: Dataset({
+    features: ['input_ids', 'attention_mask'],
+    num_rows: 600
+})
+/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
+  warnings.warn(
+[2024-10-30 22:29:54,088] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2024-10-30 22:30:03,508] [INFO] [comm.py:652:init_distributed] cdb=None
+Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination
+Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...
+Loading extension module cpu_adam...
+Time to load cpu_adam op: 5.20801043510437 seconds
+wandb: WARNING Fatal error while uploading data. Some run data will not be synced, but it will still be written to disk. Use `wandb sync` at the end of the run to try uploading.
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 219, in <module>
+    trainer.train()
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/trainer.py", line 2052, in train
+    return inner_training_loop(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/trainer.py", line 2388, in _inner_training_loop
+    tr_loss_step = self.training_step(model, inputs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/trainer.py", line 3518, in training_step
+    self.accelerator.backward(loss, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/accelerate/accelerator.py", line 2238, in backward
+    self.deepspeed_engine_wrapped.backward(loss, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/accelerate/utils/deepspeed.py", line 186, in backward
+    self.engine.backward(loss, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/utils/nvtx.py", line 18, in wrapped_fn
+    ret_val = func(*args, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/runtime/engine.py", line 2020, in backward
+    self.optimizer.backward(loss, retain_graph=retain_graph)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/runtime/zero/stage_1_and_2.py", line 2063, in backward
+    self.loss_scaler.backward(loss.float(), retain_graph=retain_graph)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/runtime/fp16/loss_scaler.py", line 63, in backward
+    scaled_loss.backward(retain_graph=retain_graph)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/torch/_tensor.py", line 487, in backward
+    torch.autograd.backward(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/torch/autograd/__init__.py", line 200, in backward
+    Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
+KeyboardInterrupt

wandb/run-20241030_222932-9ywqkbmn/files/requirements.txt

@@ -0,0 +1,147 @@
+funcsigs==1.0.2
+sentry-sdk==2.17.0
+multiprocess==0.70.16
+numpy==1.26.2
+pluralizer==1.2.0
+debugpy==1.6.7
+nvidia-cudnn-cu11==8.5.0.96
+deepspeed==0.15.2
+data==0.4
+pandas==2.1.3
+tomli==2.0.1
+charset-normalizer==3.3.2
+attrs==24.2.0
+aiosignal==1.3.1
+fsspec==2023.10.0
+nvidia-cusparse-cu11==11.7.4.91
+zipp==3.12.0
+mypy-extensions==1.0.0
+datasets==3.0.1
+joblib==1.3.2
+hjson==3.1.0
+traitlets==5.7.1
+stack-data==0.6.0
+transformers==4.45.1
+sympy==1.11.1
+Pygments==2.15.0
+docker-pycreds==0.4.0
+dill==0.3.8
+wheel==0.44.0
+prompt-toolkit==3.0.30
+parso==0.8.3
+ipykernel==6.23.1
+pyarrow==17.0.0
+certifi==2023.11.17
+nvidia-cufft-cu11==10.9.0.58
+six==1.16.0
+pydantic==2.9.2
+click==8.1.7
+nest-asyncio==1.5.6
+gmpy2==2.1.0
+matplotlib==3.8.2
+scipy==1.11.4
+typing_extensions==4.12.2
+statsmodels==0.14.0
+huggingface-hub==0.25.0
+frozenlist==1.4.1
+gpustat==1.1.1
+nvidia-nvtx-cu11==11.7.91
+safetensors==0.4.5
+stanza==1.9.2
+decorator==5.1.1
+seaborn==0.13.0
+sentencepiece==0.2.0
+PyYAML==6.0.1
+black==24.8.0
+protobuf==4.25.1
+pickleshare==0.7.5
+peft==0.13.0
+triton==2.0.0
+nvidia-cuda-runtime-cu11==11.7.99
+Jinja2==3.1.2
+nvidia-cusolver-cu11==11.4.0.1
+executing==1.2.0
+jupyter_client==8.1.0
+pluggy==1.3.0
+cmake==3.30.3
+pytz==2023.3.post1
+aiohappyeyeballs==2.4.2
+kiwisolver==1.4.5
+py-cpuinfo==9.0.0
+Pillow==10.1.0
+ptyprocess==0.7.0
+importlib_resources==6.4.5
+GitPython==3.1.43
+importlib-metadata==6.0.0
+iniconfig==2.0.0
+scikit-learn==1.3.2
+exceptiongroup==1.1.0
+networkx==2.8.6
+accelerate==1.0.0
+nltk==3.8.1
+shutilwhich==1.1.0
+fonttools==4.45.1
+future==0.18.3
+aiohttp==3.10.6
+wcwidth==0.2.5
+idna==3.6
+filelock==3.12.2
+pathspec==0.12.1
+jupyter_core==5.1.0
+lit==18.1.8
+nvidia-curand-cu11==10.2.10.91
+nvidia-cublas-cu11==11.10.3.66
+nvidia-ml-py==12.560.30
+msgpack==1.1.0
+python-dateutil==2.8.2
+blessed==1.20.0
+packaging==23.0
+gitdb==4.0.11
+yarl==1.13.0
+emoji==2.8.0
+tzdata==2023.3
+cycler==0.12.1
+tornado==6.2
+backcall==0.2.0
+plotnine==0.12.4
+ninja==1.11.1.1
+latex==0.7.0
+wandb==0.18.5
+setproctitle==1.3.3
+threadpoolctl==3.2.0
+requests==2.32.3
+pyparsing==3.1.1
+smmap==5.0.1
+pyzmq==23.0.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+matplotlib-inline==0.1.6
+latexcodec==1.0.0
+ipython==8.0.0
+patsy==0.5.3
+contourpy==1.2.0
+multidict==6.1.0
+mizani==0.9.3
+urllib3==2.1.0
+tokenizers==0.20.0
+MarkupSafe==2.1.2
+pip==24.2
+pexpect==4.8.0
+tqdm==4.66.5
+jedi==0.18.2
+pydantic_core==2.23.4
+tempdir==0.7.1
+mpmath==1.2.1
+setuptools==72.1.0
+pytest==7.4.3
+pure-eval==0.2.2
+psutil==5.9.1
+comm==0.1.2
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+regex==2023.10.3
+platformdirs==2.5.2
+asttokens==2.2.1
+torch==2.0.0
+nvidia-nccl-cu11==2.14.3
+xxhash==3.5.0

wandb/run-20241030_222932-9ywqkbmn/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-10-31T02:29:32.391009Z",
+  "args":  [
+    "--perturbation",
+    "reverse_control",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "3",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1710969503744"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241030_222932-9ywqkbmn/logs/debug-internal.log

@@ -0,0 +1,10 @@
+{"time":"2024-10-30T22:29:32.393788169-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-10-30T22:29:32.393799779-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_222932-9ywqkbmn/logs/debug-core.log"}
+{"time":"2024-10-30T22:29:32.499884149-04:00","level":"INFO","msg":"created new stream","id":"9ywqkbmn"}
+{"time":"2024-10-30T22:29:32.499913889-04:00","level":"INFO","msg":"stream: started","id":"9ywqkbmn"}
+{"time":"2024-10-30T22:29:32.499971429-04:00","level":"INFO","msg":"sender: started","stream_id":"9ywqkbmn"}
+{"time":"2024-10-30T22:29:32.499955369-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"9ywqkbmn"}}
+{"time":"2024-10-30T22:29:32.499937499-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"9ywqkbmn"}}
+{"time":"2024-10-30T22:29:32.702609686-04:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-10-30T22:56:33.124274072-04:00","level":"ERROR","msg":"HTTP error","status":404,"method":"POST","url":"https://api.wandb.ai/files/yaning1001-dartmouth-college/impossible_llm_reverse/9ywqkbmn/file_stream"}
+{"time":"2024-10-30T22:56:33.124562333-04:00","level":"ERROR+4","msg":"filestream: fatal error: filestream: failed to upload: 404 Not Found path=files/yaning1001-dartmouth-college/impossible_llm_reverse/9ywqkbmn/file_stream: {\"error\":\"run impossible_llm_reverse/9ywqkbmn not found while streaming file\"}"}

wandb/run-20241030_222932-9ywqkbmn/logs/debug.log

@@ -0,0 +1,26 @@
+2024-10-30 22:29:32,389 INFO    MainThread:447699 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-10-30 22:29:32,389 INFO    MainThread:447699 [wandb_setup.py:_flush():79] Configure stats pid to 447699
+2024-10-30 22:29:32,389 INFO    MainThread:447699 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-10-30 22:29:32,389 INFO    MainThread:447699 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-10-30 22:29:32,389 INFO    MainThread:447699 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-30 22:29:32,389 INFO    MainThread:447699 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-30 22:29:32,389 INFO    MainThread:447699 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-10-30 22:29:32,389 INFO    MainThread:447699 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-30 22:29:32,389 INFO    MainThread:447699 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_222932-9ywqkbmn/logs/debug.log
+2024-10-30 22:29:32,389 INFO    MainThread:447699 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_222932-9ywqkbmn/logs/debug-internal.log
+2024-10-30 22:29:32,389 INFO    MainThread:447699 [wandb_init.py:init():621] calling init triggers
+2024-10-30 22:29:32,389 INFO    MainThread:447699 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-10-30 22:29:32,389 INFO    MainThread:447699 [wandb_init.py:init():671] starting backend
+2024-10-30 22:29:32,389 INFO    MainThread:447699 [wandb_init.py:init():675] sending inform_init request
+2024-10-30 22:29:32,390 INFO    MainThread:447699 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-30 22:29:32,390 INFO    MainThread:447699 [wandb_init.py:init():688] backend started and connected
+2024-10-30 22:29:32,393 INFO    MainThread:447699 [wandb_init.py:init():783] updated telemetry
+2024-10-30 22:29:32,428 INFO    MainThread:447699 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-10-30 22:29:32,699 INFO    MainThread:447699 [wandb_init.py:init():867] starting run threads in backend
+2024-10-30 22:29:32,822 INFO    MainThread:447699 [wandb_run.py:_console_start():2463] atexit reg
+2024-10-30 22:29:32,822 INFO    MainThread:447699 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-10-30 22:29:32,822 INFO    MainThread:447699 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-10-30 22:29:32,822 INFO    MainThread:447699 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-10-30 22:29:32,825 INFO    MainThread:447699 [wandb_init.py:init():911] run started, returning control to user process
+2024-10-30 22:29:32,825 INFO    MainThread:447699 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_control', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0}

wandb/run-20241030_233740-anh3ext7/run-anh3ext7.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a17fba48b8dc90c68efeeedac74fa5591567e42aed8669f6fb3f81eca290fc4
+size 851968

wandb/run-20241031_002020-9dsx181l/files/config.yaml

@@ -0,0 +1,50 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.9.19
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "3":
+                - 2
+                - 13
+                - 23
+                - 55
+            "4": 3.9.19
+            "5": 0.18.5
+            "6": 4.45.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+batch_size:
+    value: 3
+epoch:
+    value: 6
+lr:
+    value: 1e-05
+perturbation:
+    value: reverse_full
+seed:
+    value: 0
+train_set:
+    value: 10M

wandb/run-20241031_002020-9dsx181l/files/output.log

@@ -0,0 +1,14 @@
+Downloading shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [02:33<00:00, 76.65s/it]
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:05<00:00,  2.86s/it]
+tokenized_valid: Dataset({
+    features: ['input_ids', 'attention_mask'],
+    num_rows: 600
+})
+/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
+  warnings.warn(
+[2024-10-31 00:23:02,520] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2024-10-31 00:23:12,314] [INFO] [comm.py:652:init_distributed] cdb=None
+Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination
+Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...
+Loading extension module cpu_adam...
+Time to load cpu_adam op: 5.378313779830933 seconds

wandb/run-20241031_002020-9dsx181l/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-10-31T04:20:20.228877Z",
+  "args":  [
+    "--perturbation",
+    "reverse_full",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "6",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1735150157824"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241031_002020-9dsx181l/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":32219}}

wandb/run-20241031_002020-9dsx181l/logs/debug-internal.log

@@ -0,0 +1,20 @@
+{"time":"2024-10-31T00:20:20.230986033-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-10-31T00:20:20.230998283-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_002020-9dsx181l/logs/debug-core.log"}
+{"time":"2024-10-31T00:20:20.3390583-04:00","level":"INFO","msg":"created new stream","id":"9dsx181l"}
+{"time":"2024-10-31T00:20:20.33908557-04:00","level":"INFO","msg":"stream: started","id":"9dsx181l"}
+{"time":"2024-10-31T00:20:20.339125211-04:00","level":"INFO","msg":"sender: started","stream_id":"9dsx181l"}
+{"time":"2024-10-31T00:20:20.339125611-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"9dsx181l"}}
+{"time":"2024-10-31T00:20:20.339107291-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"9dsx181l"}}
+{"time":"2024-10-31T00:20:21.387128777-04:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-10-31T00:33:29.413699574-04:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-10-31T01:06:51.697838739-04:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/yaning1001-dartmouth-college/impossible_llm_reverse/9dsx181l/file_stream"}
+{"time":"2024-10-31T02:54:31.112693619-04:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-10-31T09:17:19.827015198-04:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-10-31T09:17:19.903814406-04:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-10-31T09:17:20.764824765-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"saving job artifact","runtime_seconds":0.636835622,"subtasks":[{"desc":"wandb-job.json","runtime_seconds":0.083545736,"progress":"563B/563B"}]}],"total_operations":1}}
+{"time":"2024-10-31T09:17:21.374711842-04:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-10-31T09:17:22.48623333-04:00","level":"INFO","msg":"stream: closing","id":"9dsx181l"}
+{"time":"2024-10-31T09:17:22.48628913-04:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"9dsx181l"}}
+{"time":"2024-10-31T09:17:22.486352361-04:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"9dsx181l"}}
+{"time":"2024-10-31T09:17:22.486440691-04:00","level":"INFO","msg":"sender: closed","stream_id":"9dsx181l"}
+{"time":"2024-10-31T09:17:22.486608802-04:00","level":"INFO","msg":"stream: closed","id":"9dsx181l"}

wandb/run-20241031_002020-9dsx181l/logs/debug.log

@@ -0,0 +1,33 @@
+2024-10-31 00:20:20,226 INFO    MainThread:484458 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-10-31 00:20:20,226 INFO    MainThread:484458 [wandb_setup.py:_flush():79] Configure stats pid to 484458
+2024-10-31 00:20:20,226 INFO    MainThread:484458 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-10-31 00:20:20,226 INFO    MainThread:484458 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-10-31 00:20:20,226 INFO    MainThread:484458 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-31 00:20:20,226 INFO    MainThread:484458 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-31 00:20:20,226 INFO    MainThread:484458 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-10-31 00:20:20,226 INFO    MainThread:484458 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-31 00:20:20,226 INFO    MainThread:484458 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_002020-9dsx181l/logs/debug.log
+2024-10-31 00:20:20,226 INFO    MainThread:484458 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_002020-9dsx181l/logs/debug-internal.log
+2024-10-31 00:20:20,226 INFO    MainThread:484458 [wandb_init.py:init():621] calling init triggers
+2024-10-31 00:20:20,227 INFO    MainThread:484458 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-10-31 00:20:20,227 INFO    MainThread:484458 [wandb_init.py:init():671] starting backend
+2024-10-31 00:20:20,227 INFO    MainThread:484458 [wandb_init.py:init():675] sending inform_init request
+2024-10-31 00:20:20,228 INFO    MainThread:484458 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-31 00:20:20,228 INFO    MainThread:484458 [wandb_init.py:init():688] backend started and connected
+2024-10-31 00:20:20,231 INFO    MainThread:484458 [wandb_init.py:init():783] updated telemetry
+2024-10-31 00:20:20,257 INFO    MainThread:484458 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-10-31 00:20:21,383 INFO    MainThread:484458 [wandb_init.py:init():867] starting run threads in backend
+2024-10-31 00:20:21,491 INFO    MainThread:484458 [wandb_run.py:_console_start():2463] atexit reg
+2024-10-31 00:20:21,491 INFO    MainThread:484458 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-10-31 00:20:21,491 INFO    MainThread:484458 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-10-31 00:20:21,491 INFO    MainThread:484458 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-10-31 00:20:21,493 INFO    MainThread:484458 [wandb_init.py:init():911] run started, returning control to user process
+2024-10-31 00:20:21,493 INFO    MainThread:484458 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_full', 'train_set': '10M', 'batch_size': 3, 'epoch': 6, 'seed': 0, 'lr': 1e-05}
+2024-10-31 09:17:19,684 INFO    MainThread:484458 [wandb_run.py:_finish():2158] finishing run yaning1001-dartmouth-college/impossible_llm_reverse/9dsx181l
+2024-10-31 09:17:19,700 INFO    MainThread:484458 [wandb_run.py:_atexit_cleanup():2426] got exitcode: 0
+2024-10-31 09:17:19,748 INFO    MainThread:484458 [wandb_run.py:_restore():2408] restore
+2024-10-31 09:17:19,749 INFO    MainThread:484458 [wandb_run.py:_restore():2414] restore done
+2024-10-31 09:17:22,476 INFO    MainThread:484458 [wandb_run.py:_footer_history_summary_info():3975] rendering history
+2024-10-31 09:17:22,477 INFO    MainThread:484458 [wandb_run.py:_footer_history_summary_info():4007] rendering summary
+2024-10-31 09:17:22,485 INFO    MainThread:484458 [wandb_run.py:_footer_sync_info():3934] logging synced files

wandb/run-20241031_002020-ifaaozhq/run-ifaaozhq.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:293372d60a003dac0b8de9085452a587030666d589db8634f4a67c6a2421fe37
+size 19441485

wandb/run-20241031_114700-78zg7gu4/run-78zg7gu4.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb04729382de853ab3db6eac1c534b065a0869935727dff06395a2595d54bb08
+size 1015808

wandb/run-20241031_122114-lrucy0u8/files/config.yaml

@@ -0,0 +1,50 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.9.19
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "3":
+                - 2
+                - 13
+                - 23
+                - 55
+            "4": 3.9.19
+            "5": 0.18.5
+            "6": 4.45.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+batch_size:
+    value: 3
+epoch:
+    value: 6
+lr:
+    value: 5e-06
+perturbation:
+    value: reverse_full
+seed:
+    value: 0
+train_set:
+    value: 10M

wandb/run-20241031_122114-lrucy0u8/files/output.log

@@ -0,0 +1,16 @@
+model-00002-of-00002.safetensors: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████| 1.46G/1.46G [00:34<00:00, 42.5MB/s]
+Downloading shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [02:08<00:00, 64.46s/it]
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:04<00:00,  2.42s/it]
+generation_config.json: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 185/185 [00:00<00:00, 45.9kB/s]
+tokenized_valid: Dataset({
+    features: ['input_ids', 'attention_mask'],
+    num_rows: 600
+})
+/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
+  warnings.warn(
+[2024-10-31 12:23:30,557] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2024-10-31 12:23:39,131] [INFO] [comm.py:652:init_distributed] cdb=None
+Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination
+Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...
+Loading extension module cpu_adam...
+Time to load cpu_adam op: 5.044705629348755 seconds

wandb/run-20241031_122114-lrucy0u8/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-10-31T16:21:14.323301Z",
+  "args":  [
+    "--perturbation",
+    "reverse_full",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "6",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1753159962624"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241031_122114-lrucy0u8/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":32015}}

wandb/run-20241031_122114-lrucy0u8/logs/debug-internal.log

@@ -0,0 +1,21 @@
+{"time":"2024-10-31T12:21:14.326203914-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-10-31T12:21:14.326215774-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_122114-lrucy0u8/logs/debug-core.log"}
+{"time":"2024-10-31T12:21:14.533704192-04:00","level":"INFO","msg":"created new stream","id":"lrucy0u8"}
+{"time":"2024-10-31T12:21:14.533741152-04:00","level":"INFO","msg":"stream: started","id":"lrucy0u8"}
+{"time":"2024-10-31T12:21:14.533862082-04:00","level":"INFO","msg":"sender: started","stream_id":"lrucy0u8"}
+{"time":"2024-10-31T12:21:14.533816152-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"lrucy0u8"}}
+{"time":"2024-10-31T12:21:14.533886283-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"lrucy0u8"}}
+{"time":"2024-10-31T12:21:14.780003964-04:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-10-31T13:50:47.214014934-04:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-10-31T13:54:49.649487316-04:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/yaning1001-dartmouth-college/impossible_llm_reverse/lrucy0u8/file_stream"}
+{"time":"2024-10-31T18:26:25.086008056-04:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-10-31T18:43:03.172843582-04:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-10-31T21:14:49.439335738-04:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-10-31T21:14:49.510447359-04:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-10-31T21:14:50.121147297-04:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-10-31T21:14:50.241983264-04:00","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2024-10-31T21:14:51.422713566-04:00","level":"INFO","msg":"stream: closing","id":"lrucy0u8"}
+{"time":"2024-10-31T21:14:51.422746816-04:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"lrucy0u8"}}
+{"time":"2024-10-31T21:14:51.422772786-04:00","level":"INFO","msg":"sender: closed","stream_id":"lrucy0u8"}
+{"time":"2024-10-31T21:14:51.422771376-04:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"lrucy0u8"}}
+{"time":"2024-10-31T21:14:51.422887797-04:00","level":"INFO","msg":"stream: closed","id":"lrucy0u8"}

wandb/run-20241031_122114-lrucy0u8/logs/debug.log

@@ -0,0 +1,33 @@
+2024-10-31 12:21:14,321 INFO    MainThread:558433 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-10-31 12:21:14,321 INFO    MainThread:558433 [wandb_setup.py:_flush():79] Configure stats pid to 558433
+2024-10-31 12:21:14,321 INFO    MainThread:558433 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-10-31 12:21:14,321 INFO    MainThread:558433 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-10-31 12:21:14,321 INFO    MainThread:558433 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-31 12:21:14,321 INFO    MainThread:558433 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-31 12:21:14,321 INFO    MainThread:558433 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-10-31 12:21:14,321 INFO    MainThread:558433 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-31 12:21:14,321 INFO    MainThread:558433 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_122114-lrucy0u8/logs/debug.log
+2024-10-31 12:21:14,321 INFO    MainThread:558433 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_122114-lrucy0u8/logs/debug-internal.log
+2024-10-31 12:21:14,321 INFO    MainThread:558433 [wandb_init.py:init():621] calling init triggers
+2024-10-31 12:21:14,321 INFO    MainThread:558433 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-10-31 12:21:14,321 INFO    MainThread:558433 [wandb_init.py:init():671] starting backend
+2024-10-31 12:21:14,321 INFO    MainThread:558433 [wandb_init.py:init():675] sending inform_init request
+2024-10-31 12:21:14,322 INFO    MainThread:558433 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-31 12:21:14,323 INFO    MainThread:558433 [wandb_init.py:init():688] backend started and connected
+2024-10-31 12:21:14,327 INFO    MainThread:558433 [wandb_init.py:init():783] updated telemetry
+2024-10-31 12:21:14,360 INFO    MainThread:558433 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-10-31 12:21:14,776 INFO    MainThread:558433 [wandb_init.py:init():867] starting run threads in backend
+2024-10-31 12:21:14,865 INFO    MainThread:558433 [wandb_run.py:_console_start():2463] atexit reg
+2024-10-31 12:21:14,865 INFO    MainThread:558433 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-10-31 12:21:14,865 INFO    MainThread:558433 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-10-31 12:21:14,865 INFO    MainThread:558433 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-10-31 12:21:14,867 INFO    MainThread:558433 [wandb_init.py:init():911] run started, returning control to user process
+2024-10-31 12:21:14,867 INFO    MainThread:558433 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_full', 'train_set': '10M', 'batch_size': 3, 'epoch': 6, 'seed': 0, 'lr': 5e-06}
+2024-10-31 21:14:49,288 INFO    MainThread:558433 [wandb_run.py:_finish():2158] finishing run yaning1001-dartmouth-college/impossible_llm_reverse/lrucy0u8
+2024-10-31 21:14:49,400 INFO    MainThread:558433 [wandb_run.py:_atexit_cleanup():2426] got exitcode: 0
+2024-10-31 21:14:49,400 INFO    MainThread:558433 [wandb_run.py:_restore():2408] restore
+2024-10-31 21:14:49,400 INFO    MainThread:558433 [wandb_run.py:_restore():2414] restore done
+2024-10-31 21:14:51,244 INFO    MainThread:558433 [wandb_run.py:_footer_history_summary_info():3975] rendering history
+2024-10-31 21:14:51,244 INFO    MainThread:558433 [wandb_run.py:_footer_history_summary_info():4007] rendering summary
+2024-10-31 21:14:51,421 INFO    MainThread:558433 [wandb_run.py:_footer_sync_info():3934] logging synced files

wandb/run-20241101_094656-o5p0xd39/files/output.log

@@ -0,0 +1,16 @@
+Loading checkpoint shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:05<00:00,  2.58s/it]
+tokenized_valid: Dataset({
+    features: ['input_ids', 'attention_mask'],
+    num_rows: 600
+})
+/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
+  warnings.warn(
+[2024-11-01 09:47:03,530] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2024-11-01 09:47:11,861] [INFO] [comm.py:652:init_distributed] cdb=None
+Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination
+Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...
+Emitting ninja build file /home/chunhui/.cache/torch_extensions/py39_cu117/cpu_adam/build.ninja...
+Building extension module cpu_adam...
+Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N)
+Loading extension module cpu_adam...
+Time to load cpu_adam op: 4.762843608856201 seconds

wandb/run-20241101_094656-o5p0xd39/files/requirements.txt

@@ -0,0 +1,147 @@
+funcsigs==1.0.2
+sentry-sdk==2.17.0
+multiprocess==0.70.16
+numpy==1.26.2
+pluralizer==1.2.0
+debugpy==1.6.7
+nvidia-cudnn-cu11==8.5.0.96
+deepspeed==0.15.2
+data==0.4
+pandas==2.1.3
+tomli==2.0.1
+charset-normalizer==3.3.2
+attrs==24.2.0
+aiosignal==1.3.1
+fsspec==2023.10.0
+nvidia-cusparse-cu11==11.7.4.91
+zipp==3.12.0
+mypy-extensions==1.0.0
+datasets==3.0.1
+joblib==1.3.2
+hjson==3.1.0
+traitlets==5.7.1
+stack-data==0.6.0
+transformers==4.45.1
+sympy==1.11.1
+Pygments==2.15.0
+docker-pycreds==0.4.0
+dill==0.3.8
+wheel==0.44.0
+prompt-toolkit==3.0.30
+parso==0.8.3
+ipykernel==6.23.1
+pyarrow==17.0.0
+certifi==2023.11.17
+nvidia-cufft-cu11==10.9.0.58
+six==1.16.0
+pydantic==2.9.2
+click==8.1.7
+nest-asyncio==1.5.6
+gmpy2==2.1.0
+matplotlib==3.8.2
+scipy==1.11.4
+typing_extensions==4.12.2
+statsmodels==0.14.0
+huggingface-hub==0.25.0
+frozenlist==1.4.1
+gpustat==1.1.1
+nvidia-nvtx-cu11==11.7.91
+safetensors==0.4.5
+stanza==1.9.2
+decorator==5.1.1
+seaborn==0.13.0
+sentencepiece==0.2.0
+PyYAML==6.0.1
+black==24.8.0
+protobuf==4.25.1
+pickleshare==0.7.5
+peft==0.13.0
+triton==2.0.0
+nvidia-cuda-runtime-cu11==11.7.99
+Jinja2==3.1.2
+nvidia-cusolver-cu11==11.4.0.1
+executing==1.2.0
+jupyter_client==8.1.0
+pluggy==1.3.0
+cmake==3.30.3
+pytz==2023.3.post1
+aiohappyeyeballs==2.4.2
+kiwisolver==1.4.5
+py-cpuinfo==9.0.0
+Pillow==10.1.0
+ptyprocess==0.7.0
+importlib_resources==6.4.5
+GitPython==3.1.43
+importlib-metadata==6.0.0
+iniconfig==2.0.0
+scikit-learn==1.3.2
+exceptiongroup==1.1.0
+networkx==2.8.6
+accelerate==1.0.0
+nltk==3.8.1
+shutilwhich==1.1.0
+fonttools==4.45.1
+future==0.18.3
+aiohttp==3.10.6
+wcwidth==0.2.5
+idna==3.6
+filelock==3.12.2
+pathspec==0.12.1
+jupyter_core==5.1.0
+lit==18.1.8
+nvidia-curand-cu11==10.2.10.91
+nvidia-cublas-cu11==11.10.3.66
+nvidia-ml-py==12.560.30
+msgpack==1.1.0
+python-dateutil==2.8.2
+blessed==1.20.0
+packaging==23.0
+gitdb==4.0.11
+yarl==1.13.0
+emoji==2.8.0
+tzdata==2023.3
+cycler==0.12.1
+tornado==6.2
+backcall==0.2.0
+plotnine==0.12.4
+ninja==1.11.1.1
+latex==0.7.0
+wandb==0.18.5
+setproctitle==1.3.3
+threadpoolctl==3.2.0
+requests==2.32.3
+pyparsing==3.1.1
+smmap==5.0.1
+pyzmq==23.0.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+matplotlib-inline==0.1.6
+latexcodec==1.0.0
+ipython==8.0.0
+patsy==0.5.3
+contourpy==1.2.0
+multidict==6.1.0
+mizani==0.9.3
+urllib3==2.1.0
+tokenizers==0.20.0
+MarkupSafe==2.1.2
+pip==24.2
+pexpect==4.8.0
+tqdm==4.66.5
+jedi==0.18.2
+pydantic_core==2.23.4
+tempdir==0.7.1
+mpmath==1.2.1
+setuptools==72.1.0
+pytest==7.4.3
+pure-eval==0.2.2
+psutil==5.9.1
+comm==0.1.2
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+regex==2023.10.3
+platformdirs==2.5.2
+asttokens==2.2.1
+torch==2.0.0
+nvidia-nccl-cu11==2.14.3
+xxhash==3.5.0