Add files using upload-large-folder tool

wandb/run-20241030_010641-0a117nzt/files/config.yaml

@@ -0,0 +1,47 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.9.19
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "3":
+                - 13
+                - 23
+                - 55
+            "4": 3.9.19
+            "5": 0.18.5
+            "6": 4.45.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+batch_size:
+    value: 3
+epoch:
+    value: 7
+perturbation:
+    value: reverse_control
+seed:
+    value: 0
+train_set:
+    value: 10M

wandb/run-20241030_010641-0a117nzt/files/wandb-metadata.json

@@ -0,0 +1,29 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-10-30T05:06:41.506481Z",
+  "args":  [
+    "--perturbation",
+    "reverse_control",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "7",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py"
+}

wandb/run-20241030_010641-0a117nzt/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":1}}

wandb/run-20241030_010641-0a117nzt/logs/debug-internal.log

@@ -0,0 +1,16 @@
+{"time":"2024-10-30T01:06:41.508846039-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-10-30T01:06:41.508860409-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_010641-0a117nzt/logs/debug-core.log"}
+{"time":"2024-10-30T01:06:41.614626711-04:00","level":"INFO","msg":"created new stream","id":"0a117nzt"}
+{"time":"2024-10-30T01:06:41.614656671-04:00","level":"INFO","msg":"stream: started","id":"0a117nzt"}
+{"time":"2024-10-30T01:06:41.614698271-04:00","level":"INFO","msg":"sender: started","stream_id":"0a117nzt"}
+{"time":"2024-10-30T01:06:41.614675501-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"0a117nzt"}}
+{"time":"2024-10-30T01:06:41.614701901-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"0a117nzt"}}
+{"time":"2024-10-30T01:06:43.050075385-04:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-10-30T01:06:43.148263865-04:00","level":"INFO","msg":"stream: closing","id":"0a117nzt"}
+{"time":"2024-10-30T01:06:43.148301525-04:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-10-30T01:06:43.23979748-04:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-10-30T01:06:43.642158405-04:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-10-30T01:06:43.754139519-04:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"0a117nzt"}}
+{"time":"2024-10-30T01:06:43.754179299-04:00","level":"INFO","msg":"sender: closed","stream_id":"0a117nzt"}
+{"time":"2024-10-30T01:06:43.754174569-04:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"0a117nzt"}}
+{"time":"2024-10-30T01:06:43.754253939-04:00","level":"INFO","msg":"stream: closed","id":"0a117nzt"}

wandb/run-20241030_010641-0a117nzt/logs/debug.log

@@ -0,0 +1,27 @@
+2024-10-30 01:06:41,504 INFO    MainThread:321597 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-10-30 01:06:41,504 INFO    MainThread:321597 [wandb_setup.py:_flush():79] Configure stats pid to 321597
+2024-10-30 01:06:41,504 INFO    MainThread:321597 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-10-30 01:06:41,504 INFO    MainThread:321597 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-10-30 01:06:41,504 INFO    MainThread:321597 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-30 01:06:41,505 INFO    MainThread:321597 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-30 01:06:41,505 INFO    MainThread:321597 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-10-30 01:06:41,505 INFO    MainThread:321597 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-30 01:06:41,505 INFO    MainThread:321597 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_010641-0a117nzt/logs/debug.log
+2024-10-30 01:06:41,505 INFO    MainThread:321597 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_010641-0a117nzt/logs/debug-internal.log
+2024-10-30 01:06:41,505 INFO    MainThread:321597 [wandb_init.py:init():621] calling init triggers
+2024-10-30 01:06:41,505 INFO    MainThread:321597 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-10-30 01:06:41,505 INFO    MainThread:321597 [wandb_init.py:init():671] starting backend
+2024-10-30 01:06:41,505 INFO    MainThread:321597 [wandb_init.py:init():675] sending inform_init request
+2024-10-30 01:06:41,506 INFO    MainThread:321597 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-30 01:06:41,506 INFO    MainThread:321597 [wandb_init.py:init():688] backend started and connected
+2024-10-30 01:06:41,509 INFO    MainThread:321597 [wandb_init.py:init():783] updated telemetry
+2024-10-30 01:06:41,529 INFO    MainThread:321597 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-10-30 01:06:43,047 INFO    MainThread:321597 [wandb_init.py:init():867] starting run threads in backend
+2024-10-30 01:06:43,145 INFO    MainThread:321597 [wandb_run.py:_console_start():2463] atexit reg
+2024-10-30 01:06:43,145 INFO    MainThread:321597 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-10-30 01:06:43,145 INFO    MainThread:321597 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-10-30 01:06:43,145 INFO    MainThread:321597 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-10-30 01:06:43,147 INFO    MainThread:321597 [wandb_init.py:init():911] run started, returning control to user process
+2024-10-30 01:06:43,147 INFO    MainThread:321597 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_control', 'train_set': '10M', 'batch_size': 3, 'epoch': 7, 'seed': 0}
+2024-10-30 01:06:43,148 WARNING MsgRouterThr:321597 [router.py:message_loop():77] message_loop has been closed

wandb/run-20241030_011013-8qrwqf2b/files/output.log

@@ -0,0 +1,6 @@
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 165, in <module>
+    valid_dataset = dataset['validation']
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/dataset_dict.py", line 72, in __getitem__
+    return super().__getitem__(k)
+KeyError: 'validation'

wandb/run-20241030_011013-8qrwqf2b/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":8}}

wandb/run-20241030_011013-8qrwqf2b/logs/debug-internal.log

@@ -0,0 +1,16 @@
+{"time":"2024-10-30T01:10:13.811924092-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-10-30T01:10:13.811935262-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_011013-8qrwqf2b/logs/debug-core.log"}
+{"time":"2024-10-30T01:10:13.919016219-04:00","level":"INFO","msg":"created new stream","id":"8qrwqf2b"}
+{"time":"2024-10-30T01:10:13.919054309-04:00","level":"INFO","msg":"stream: started","id":"8qrwqf2b"}
+{"time":"2024-10-30T01:10:13.91913415-04:00","level":"INFO","msg":"sender: started","stream_id":"8qrwqf2b"}
+{"time":"2024-10-30T01:10:13.919078679-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"8qrwqf2b"}}
+{"time":"2024-10-30T01:10:13.91913301-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"8qrwqf2b"}}
+{"time":"2024-10-30T01:10:14.203333759-04:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-10-30T01:10:21.984884451-04:00","level":"INFO","msg":"stream: closing","id":"8qrwqf2b"}
+{"time":"2024-10-30T01:10:21.984959852-04:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-10-30T01:10:21.985933548-04:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-10-30T01:10:22.321607177-04:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-10-30T01:10:22.49142805-04:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"8qrwqf2b"}}
+{"time":"2024-10-30T01:10:22.49146284-04:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"8qrwqf2b"}}
+{"time":"2024-10-30T01:10:22.49147302-04:00","level":"INFO","msg":"sender: closed","stream_id":"8qrwqf2b"}
+{"time":"2024-10-30T01:10:22.49151008-04:00","level":"INFO","msg":"stream: closed","id":"8qrwqf2b"}

wandb/run-20241030_011013-8qrwqf2b/logs/debug.log

@@ -0,0 +1,27 @@
+2024-10-30 01:10:13,807 INFO    MainThread:323568 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-10-30 01:10:13,807 INFO    MainThread:323568 [wandb_setup.py:_flush():79] Configure stats pid to 323568
+2024-10-30 01:10:13,807 INFO    MainThread:323568 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-10-30 01:10:13,807 INFO    MainThread:323568 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-10-30 01:10:13,807 INFO    MainThread:323568 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-30 01:10:13,807 INFO    MainThread:323568 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-30 01:10:13,807 INFO    MainThread:323568 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-10-30 01:10:13,807 INFO    MainThread:323568 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-30 01:10:13,808 INFO    MainThread:323568 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_011013-8qrwqf2b/logs/debug.log
+2024-10-30 01:10:13,808 INFO    MainThread:323568 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_011013-8qrwqf2b/logs/debug-internal.log
+2024-10-30 01:10:13,808 INFO    MainThread:323568 [wandb_init.py:init():621] calling init triggers
+2024-10-30 01:10:13,808 INFO    MainThread:323568 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-10-30 01:10:13,808 INFO    MainThread:323568 [wandb_init.py:init():671] starting backend
+2024-10-30 01:10:13,808 INFO    MainThread:323568 [wandb_init.py:init():675] sending inform_init request
+2024-10-30 01:10:13,809 INFO    MainThread:323568 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-30 01:10:13,809 INFO    MainThread:323568 [wandb_init.py:init():688] backend started and connected
+2024-10-30 01:10:13,812 INFO    MainThread:323568 [wandb_init.py:init():783] updated telemetry
+2024-10-30 01:10:13,844 INFO    MainThread:323568 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-10-30 01:10:14,200 INFO    MainThread:323568 [wandb_init.py:init():867] starting run threads in backend
+2024-10-30 01:10:14,291 INFO    MainThread:323568 [wandb_run.py:_console_start():2463] atexit reg
+2024-10-30 01:10:14,291 INFO    MainThread:323568 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-10-30 01:10:14,291 INFO    MainThread:323568 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-10-30 01:10:14,291 INFO    MainThread:323568 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-10-30 01:10:14,292 INFO    MainThread:323568 [wandb_init.py:init():911] run started, returning control to user process
+2024-10-30 01:10:14,292 INFO    MainThread:323568 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_control', 'train_set': '10M', 'batch_size': 3, 'epoch': 7, 'seed': 0}
+2024-10-30 01:10:21,985 WARNING MsgRouterThr:323568 [router.py:message_loop():77] message_loop has been closed

wandb/run-20241030_011509-46hc4g2h/files/output.log

@@ -0,0 +1,15 @@
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:04<00:00,  2.33s/it]
+Map: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 17519/17519 [00:55<00:00, 312.86 examples/s]
+Map: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 18140/18140 [00:56<00:00, 323.57 examples/s]
+tokenized_valid: Dataset({
+    features: ['input_ids', 'attention_mask'],
+    num_rows: 600
+})
+/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
+  warnings.warn(
+[2024-10-30 01:17:08,151] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2024-10-30 01:17:15,333] [INFO] [comm.py:652:init_distributed] cdb=None
+Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination
+Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...
+Loading extension module cpu_adam...
+Time to load cpu_adam op: 4.191785097122192 seconds

wandb/run-20241030_112852-av3r7rx8/logs/debug-internal.log

@@ -0,0 +1,107 @@
+{"time":"2024-10-30T11:28:52.928202044-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-10-30T11:28:52.928217684-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_112852-av3r7rx8/logs/debug-core.log"}
+{"time":"2024-10-30T11:28:53.035202578-04:00","level":"INFO","msg":"created new stream","id":"av3r7rx8"}
+{"time":"2024-10-30T11:28:53.035234758-04:00","level":"INFO","msg":"stream: started","id":"av3r7rx8"}
+{"time":"2024-10-30T11:28:53.035306888-04:00","level":"INFO","msg":"sender: started","stream_id":"av3r7rx8"}
+{"time":"2024-10-30T11:28:53.035302768-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"av3r7rx8"}}
+{"time":"2024-10-30T11:28:53.035283128-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"av3r7rx8"}}
+{"time":"2024-10-30T11:28:53.237316902-04:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-10-30T14:03:23.662768215-04:00","level":"ERROR","msg":"HTTP error","status":404,"method":"POST","url":"https://api.wandb.ai/files/yaning1001-dartmouth-college/impossible_llm_reverse/av3r7rx8/file_stream"}
+{"time":"2024-10-30T14:03:23.671100253-04:00","level":"ERROR+4","msg":"filestream: fatal error: filestream: failed to upload: 404 Not Found path=files/yaning1001-dartmouth-college/impossible_llm_reverse/av3r7rx8/file_stream: {\"error\":\"run impossible_llm_reverse/av3r7rx8 not found while streaming file\"}"}
+{"time":"2024-10-30T18:00:35.969420417-04:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-10-30T18:00:35.985259874-04:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-10-30T18:00:36.00413501-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:00:36.970397087-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":1.017823332,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:00:38.233986167-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:00:42.56959314-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:00:50.937735567-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:01:07.87865361-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:01:36.999941082-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":61.047372157,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:01:42.14950262-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:02:37.029338398-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":121.076773074,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:02:42.2181434-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:03:37.121523999-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":181.168948695,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:03:42.283494538-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:04:37.15272486-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":241.200157826,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:04:42.338284797-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:05:37.18149733-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":301.228929406,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:05:42.403305011-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:06:37.205923464-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":361.25334464,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:06:42.467736367-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:07:37.227919994-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":421.27535455,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:07:42.522828129-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:08:37.255122245-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":481.302550111,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:08:42.591448018-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:09:37.275888025-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":541.323308591,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:09:42.646782455-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:10:35.952783302-04:00","level":"WARN","msg":"sender: taking a long time","seconds":600.000251687,"work":"WorkRecord(*service_go_proto.Record_Telemetry); Control(connection_id:\"127.0.0.1:52674\")"}
+{"time":"2024-10-30T18:10:37.297919548-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":601.345344493,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:10:42.706639294-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:11:37.331253116-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":661.378679132,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:11:42.759908944-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:12:37.356310441-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":721.403746647,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:12:42.814966869-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:13:37.391653109-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":781.439081615,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:13:42.884911218-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:14:37.425098592-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":841.472520388,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:14:42.949899596-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:15:37.446427353-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":901.493859249,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:15:43.005675097-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:16:37.466452255-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":961.5138852,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:16:43.072875312-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:16:43.073018214-04:00","level":"ERROR","msg":"sender: sendConfig:","error":"api: failed sending: POST https://api.wandb.ai/graphql giving up after 21 attempt(s)"}
+{"time":"2024-10-30T18:16:43.073207477-04:00","level":"INFO","msg":"sender: succeeded after taking longer than expected","seconds":967.120781373,"work":"WorkRecord(*service_go_proto.Record_Telemetry); Control(connection_id:\"127.0.0.1:52674\")"}
+{"time":"2024-10-30T18:16:43.129772686-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:16:43.173405496-04:00","level":"ERROR","msg":"HTTP error","status":404,"method":"POST","url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:16:43.173464037-04:00","level":"ERROR","msg":"runfiles: CreateRunFiles returned error: returned error 404 Not Found: {\"errors\":[{\"message\":\"run impossible_llm_reverse/av3r7rx8 not found during createRunFiles\",\"path\":[\"createRunFiles\"]}],\"data\":{\"createRunFiles\":null}}"}
+{"time":"2024-10-30T18:16:45.568906891-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:16:49.876981993-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:16:59.135590692-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:17:17.819043918-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:17:37.496704741-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":54.423069439,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:17:54.967451159-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:18:37.527279913-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":114.453646191,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:18:55.01780239-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:19:37.558314637-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":174.484679915,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:19:55.069104494-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:20:37.584777577-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":234.511143195,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:20:55.119365993-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:21:37.608570107-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":294.534931265,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:21:55.17082767-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:22:37.631511339-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":354.557876897,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:22:55.22139184-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:23:37.662681067-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":414.589036285,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:23:55.274106938-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:24:37.684696297-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":474.611047365,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:24:55.324688042-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:25:37.707091894-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":534.633454442,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:25:55.38216525-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:26:37.732778072-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":594.65913657,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:26:43.074198907-04:00","level":"WARN","msg":"sender: taking a long time","seconds":600.000310312,"work":"WorkRecord(*service_go_proto.Request_Defer); Control(local:true  always_send:true)"}
+{"time":"2024-10-30T18:26:55.440696492-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:27:37.755666689-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":654.682007297,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:27:55.492005526-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:28:37.776371583-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":714.702731411,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:28:55.553278137-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:29:37.803304571-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":774.729659329,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:29:55.604429145-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:30:37.830574404-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":834.756934202,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:30:55.662350387-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:31:37.854749431-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":894.781096409,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:31:55.727511014-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:32:37.877695674-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":954.804047742,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:32:55.816423244-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:32:55.816545335-04:00","level":"ERROR","msg":"sender: sendConfig:","error":"api: failed sending: POST https://api.wandb.ai/graphql giving up after 21 attempt(s)"}
+{"time":"2024-10-30T18:32:55.817180741-04:00","level":"INFO","msg":"sender: succeeded after taking longer than expected","seconds":972.743306456,"work":"WorkRecord(*service_go_proto.Request_Defer); Control(local:true  always_send:true)"}
+{"time":"2024-10-30T18:32:55.916557726-04:00","level":"ERROR","msg":"HTTP error","status":404,"method":"POST","url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:32:55.916592226-04:00","level":"ERROR","msg":"runfiles: CreateRunFiles returned error: returned error 404 Not Found: {\"errors\":[{\"message\":\"run impossible_llm_reverse/av3r7rx8 not found during createRunFiles\",\"path\":[\"createRunFiles\"]}],\"data\":{\"createRunFiles\":null}}"}
+{"time":"2024-10-30T18:32:56.047553113-04:00","level":"ERROR","msg":"HTTP error","status":404,"method":"POST","url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:32:56.047914706-04:00","level":"ERROR","msg":"sender: failed to save job artifact: ArtifactSaver.createManifest: returned error 404 Not Found: {\"errors\":[{\"message\":\"failed to find run impossible_llm_reverse/av3r7rx8\",\"path\":[\"createArtifactManifest\"]}],\"data\":{\"createArtifactManifest\":null}}"}
+{"time":"2024-10-30T18:32:56.099783357-04:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-10-30T18:32:56.149719522-04:00","level":"ERROR","msg":"HTTP error","status":404,"method":"POST","url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:32:56.149778963-04:00","level":"ERROR","msg":"runfiles: CreateRunFiles returned error: returned error 404 Not Found: {\"errors\":[{\"message\":\"run impossible_llm_reverse/av3r7rx8 not found during createRunFiles\",\"path\":[\"createRunFiles\"]}],\"data\":{\"createRunFiles\":null}}"}
+{"time":"2024-10-30T18:32:57.117726892-04:00","level":"INFO","msg":"stream: closing","id":"av3r7rx8"}
+{"time":"2024-10-30T18:32:57.117766882-04:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"av3r7rx8"}}
+{"time":"2024-10-30T18:32:57.117795143-04:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"av3r7rx8"}}
+{"time":"2024-10-30T18:32:57.117827703-04:00","level":"INFO","msg":"sender: closed","stream_id":"av3r7rx8"}
+{"time":"2024-10-30T18:32:57.117911104-04:00","level":"INFO","msg":"stream: closed","id":"av3r7rx8"}

wandb/run-20241030_222932-sh4rlbgu/files/output.log

@@ -0,0 +1,239 @@
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:18<00:00,  9.39s/it]
+tokenized_valid: Dataset({
+    features: ['input_ids', 'attention_mask'],
+    num_rows: 600
+})
+/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
+  warnings.warn(
+[2024-10-30 22:29:54,147] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2024-10-30 22:30:03,558] [INFO] [comm.py:652:init_distributed] cdb=None
+[2024-10-30 22:30:03,558] [INFO] [comm.py:683:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl
+Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.
+Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination
+Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...
+Loading extension module cpu_adam...
+Time to load cpu_adam op: 5.195138931274414 seconds
+wandb: WARNING The `run_name` is currently set to the same value as `TrainingArguments.output_dir`. If this was not intended, please specify a different run name by setting the `TrainingArguments.run_name` parameter.
+                                                                                                                                                                                       
+{'loss': 2.7522, 'grad_norm': 0.0, 'learning_rate': 1e-05, 'epoch': 0.0}
+{'loss': 2.7541, 'grad_norm': 0.0, 'learning_rate': 1e-05, 'epoch': 0.0}
+{'loss': 2.8031, 'grad_norm': 0.0, 'learning_rate': 1e-05, 'epoch': 0.0}
+{'loss': 2.8062, 'grad_norm': 0.0, 'learning_rate': 1e-05, 'epoch': 0.0}
+{'loss': 2.7733, 'grad_norm': 0.0, 'learning_rate': 1e-05, 'epoch': 0.01}
+{'loss': 2.7949, 'grad_norm': 0.0, 'learning_rate': 1e-05, 'epoch': 0.01}
+{'loss': 2.7657, 'grad_norm': 0.0, 'learning_rate': 1e-05, 'epoch': 0.01}
+{'loss': 2.803, 'grad_norm': 0.0, 'learning_rate': 1e-05, 'epoch': 0.01}
+{'loss': 2.7793, 'grad_norm': 0.0, 'learning_rate': 1e-05, 'epoch': 0.01}
+{'loss': 2.7935, 'grad_norm': 0.0, 'learning_rate': 1e-05, 'epoch': 0.01}
+                                                                                                                                                                                       
+{'eval_loss': 2.8027420043945312, 'eval_runtime': 12.3412, 'eval_samples_per_second': 48.618, 'eval_steps_per_second': 1.053, 'epoch': 0.01}
+{'loss': 2.7546, 'grad_norm': 0.0, 'learning_rate': 1e-05, 'epoch': 0.01}
+{'loss': 2.7423, 'grad_norm': 0.0, 'learning_rate': 1e-05, 'epoch': 0.01}
+{'loss': 2.8057, 'grad_norm': 0.0, 'learning_rate': 1e-05, 'epoch': 0.01}
+{'loss': 2.7663, 'grad_norm': 0.0, 'learning_rate': 1e-05, 'epoch': 0.01}
+{'loss': 2.7715, 'grad_norm': 0.0, 'learning_rate': 1e-05, 'epoch': 0.02}
+{'loss': 2.751, 'grad_norm': 4.9052934646606445, 'learning_rate': 9.996577686516086e-06, 'epoch': 0.02}
+{'loss': 2.7356, 'grad_norm': 4.9052934646606445, 'learning_rate': 9.996577686516086e-06, 'epoch': 0.02}
+{'loss': 2.7975, 'grad_norm': 6.998753070831299, 'learning_rate': 9.993155373032171e-06, 'epoch': 0.02}
+{'loss': 2.6815, 'grad_norm': 6.998753070831299, 'learning_rate': 9.993155373032171e-06, 'epoch': 0.02}
+{'loss': 2.7412, 'grad_norm': 10.557236671447754, 'learning_rate': 9.989733059548255e-06, 'epoch': 0.02}
+{'eval_loss': 2.638756513595581, 'eval_runtime': 11.4482, 'eval_samples_per_second': 52.41, 'eval_steps_per_second': 1.136, 'epoch': 0.02}
+{'loss': 2.62, 'grad_norm': 4.8778886795043945, 'learning_rate': 9.98631074606434e-06, 'epoch': 0.02}
+{'loss': 2.5717, 'grad_norm': 5.405064582824707, 'learning_rate': 9.982888432580425e-06, 'epoch': 0.02}
+{'loss': 2.4702, 'grad_norm': 3.2149922847747803, 'learning_rate': 9.97946611909651e-06, 'epoch': 0.02}
+{'loss': 2.4408, 'grad_norm': 2.206319570541382, 'learning_rate': 9.976043805612596e-06, 'epoch': 0.02}
+{'loss': 2.4046, 'grad_norm': 2.7088518142700195, 'learning_rate': 9.972621492128681e-06, 'epoch': 0.03}
+{'loss': 2.4052, 'grad_norm': 2.255714178085327, 'learning_rate': 9.969199178644764e-06, 'epoch': 0.03}
+{'loss': 2.3598, 'grad_norm': 1.8110233545303345, 'learning_rate': 9.96577686516085e-06, 'epoch': 0.03}
+{'loss': 2.359, 'grad_norm': 2.3473737239837646, 'learning_rate': 9.962354551676935e-06, 'epoch': 0.03}
+{'loss': 2.3203, 'grad_norm': 1.7912501096725464, 'learning_rate': 9.95893223819302e-06, 'epoch': 0.03}
+{'loss': 2.2915, 'grad_norm': 1.74528169631958, 'learning_rate': 9.955509924709105e-06, 'epoch': 0.03}
+{'eval_loss': 2.381314277648926, 'eval_runtime': 11.4705, 'eval_samples_per_second': 52.308, 'eval_steps_per_second': 1.133, 'epoch': 0.03}
+{'loss': 2.2406, 'grad_norm': 2.214596748352051, 'learning_rate': 9.952087611225189e-06, 'epoch': 0.03}
+{'loss': 2.2711, 'grad_norm': 1.6342750787734985, 'learning_rate': 9.948665297741274e-06, 'epoch': 0.03}
+{'loss': 2.2322, 'grad_norm': 1.947365641593933, 'learning_rate': 9.945242984257358e-06, 'epoch': 0.03}
+{'loss': 2.1942, 'grad_norm': 1.863891839981079, 'learning_rate': 9.941820670773443e-06, 'epoch': 0.03}
+{'loss': 2.1649, 'grad_norm': 2.0973386764526367, 'learning_rate': 9.938398357289528e-06, 'epoch': 0.04}
+{'loss': 2.0559, 'grad_norm': 1.8396668434143066, 'learning_rate': 9.934976043805613e-06, 'epoch': 0.04}
+{'loss': 2.1748, 'grad_norm': 1.8861109018325806, 'learning_rate': 9.931553730321698e-06, 'epoch': 0.04}
+{'loss': 2.0844, 'grad_norm': 1.8722198009490967, 'learning_rate': 9.928131416837784e-06, 'epoch': 0.04}
+{'loss': 2.0759, 'grad_norm': 1.6950063705444336, 'learning_rate': 9.924709103353867e-06, 'epoch': 0.04}
+{'loss': 2.1387, 'grad_norm': 2.20597243309021, 'learning_rate': 9.921286789869952e-06, 'epoch': 0.04}
+{'eval_loss': 2.2260310649871826, 'eval_runtime': 11.4922, 'eval_samples_per_second': 52.209, 'eval_steps_per_second': 1.131, 'epoch': 0.04}
+{'loss': 2.0875, 'grad_norm': 1.525025725364685, 'learning_rate': 9.917864476386038e-06, 'epoch': 0.04}
+{'loss': 2.0144, 'grad_norm': 1.8138060569763184, 'learning_rate': 9.914442162902123e-06, 'epoch': 0.04}
+{'loss': 2.073, 'grad_norm': 1.8054825067520142, 'learning_rate': 9.911019849418208e-06, 'epoch': 0.04}
+{'loss': 2.0333, 'grad_norm': 1.8127416372299194, 'learning_rate': 9.907597535934292e-06, 'epoch': 0.05}
+{'loss': 2.0386, 'grad_norm': 1.548357605934143, 'learning_rate': 9.904175222450377e-06, 'epoch': 0.05}
+{'loss': 2.0096, 'grad_norm': 1.63664710521698, 'learning_rate': 9.900752908966462e-06, 'epoch': 0.05}
+{'loss': 2.063, 'grad_norm': 1.5788047313690186, 'learning_rate': 9.897330595482547e-06, 'epoch': 0.05}
+{'loss': 1.9989, 'grad_norm': 1.9751560688018799, 'learning_rate': 9.893908281998632e-06, 'epoch': 0.05}
+{'loss': 2.007, 'grad_norm': 1.5146485567092896, 'learning_rate': 9.890485968514718e-06, 'epoch': 0.05}
+{'loss': 1.9296, 'grad_norm': 1.671108365058899, 'learning_rate': 9.887063655030801e-06, 'epoch': 0.05}
+{'eval_loss': 2.1585757732391357, 'eval_runtime': 11.4929, 'eval_samples_per_second': 52.206, 'eval_steps_per_second': 1.131, 'epoch': 0.05}
+{'loss': 1.938, 'grad_norm': 1.7599347829818726, 'learning_rate': 9.883641341546886e-06, 'epoch': 0.05}
+{'loss': 2.0424, 'grad_norm': 1.814841866493225, 'learning_rate': 9.880219028062972e-06, 'epoch': 0.05}
+{'loss': 1.9183, 'grad_norm': 1.763486385345459, 'learning_rate': 9.876796714579057e-06, 'epoch': 0.05}
+{'loss': 1.9818, 'grad_norm': 1.9463744163513184, 'learning_rate': 9.873374401095142e-06, 'epoch': 0.06}
+{'loss': 1.9367, 'grad_norm': 1.6990312337875366, 'learning_rate': 9.869952087611226e-06, 'epoch': 0.06}
+{'loss': 1.8971, 'grad_norm': 1.7191174030303955, 'learning_rate': 9.866529774127311e-06, 'epoch': 0.06}
+{'loss': 1.9414, 'grad_norm': 1.75558340549469, 'learning_rate': 9.863107460643394e-06, 'epoch': 0.06}
+{'loss': 1.8678, 'grad_norm': 1.6302168369293213, 'learning_rate': 9.85968514715948e-06, 'epoch': 0.06}
+{'loss': 1.9358, 'grad_norm': 1.6982876062393188, 'learning_rate': 9.856262833675565e-06, 'epoch': 0.06}
+{'loss': 1.9275, 'grad_norm': 1.7876794338226318, 'learning_rate': 9.85284052019165e-06, 'epoch': 0.06}
+{'eval_loss': 2.127957820892334, 'eval_runtime': 11.513, 'eval_samples_per_second': 52.115, 'eval_steps_per_second': 1.129, 'epoch': 0.06}
+{'loss': 1.8948, 'grad_norm': 1.7176604270935059, 'learning_rate': 9.849418206707735e-06, 'epoch': 0.06}
+{'loss': 1.9117, 'grad_norm': 1.7535561323165894, 'learning_rate': 9.84599589322382e-06, 'epoch': 0.06}
+{'loss': 1.9544, 'grad_norm': 1.753197431564331, 'learning_rate': 9.842573579739904e-06, 'epoch': 0.06}
+{'loss': 1.9392, 'grad_norm': 1.7100030183792114, 'learning_rate': 9.83915126625599e-06, 'epoch': 0.07}
+{'loss': 1.8039, 'grad_norm': 1.9034301042556763, 'learning_rate': 9.835728952772074e-06, 'epoch': 0.07}
+{'loss': 1.9127, 'grad_norm': 2.095262289047241, 'learning_rate': 9.83230663928816e-06, 'epoch': 0.07}
+{'loss': 1.8438, 'grad_norm': 1.80649995803833, 'learning_rate': 9.828884325804245e-06, 'epoch': 0.07}
+{'loss': 1.9244, 'grad_norm': 1.6947416067123413, 'learning_rate': 9.825462012320328e-06, 'epoch': 0.07}
+{'loss': 1.8639, 'grad_norm': 1.8084263801574707, 'learning_rate': 9.822039698836414e-06, 'epoch': 0.07}
+{'loss': 1.9785, 'grad_norm': 1.8480068445205688, 'learning_rate': 9.818617385352499e-06, 'epoch': 0.07}
+{'eval_loss': 2.099644660949707, 'eval_runtime': 11.5197, 'eval_samples_per_second': 52.085, 'eval_steps_per_second': 1.128, 'epoch': 0.07}
+{'loss': 1.8751, 'grad_norm': 1.7712780237197876, 'learning_rate': 9.815195071868584e-06, 'epoch': 0.07}
+{'loss': 1.8161, 'grad_norm': 2.03159761428833, 'learning_rate': 9.81177275838467e-06, 'epoch': 0.07}
+{'loss': 1.8763, 'grad_norm': 1.8015327453613281, 'learning_rate': 9.808350444900755e-06, 'epoch': 0.07}
+{'loss': 1.8381, 'grad_norm': 2.4486355781555176, 'learning_rate': 9.804928131416838e-06, 'epoch': 0.08}
+{'loss': 1.85, 'grad_norm': 1.7883203029632568, 'learning_rate': 9.801505817932923e-06, 'epoch': 0.08}
+{'loss': 1.8443, 'grad_norm': 2.3537960052490234, 'learning_rate': 9.798083504449008e-06, 'epoch': 0.08}
+{'loss': 1.857, 'grad_norm': 2.174302101135254, 'learning_rate': 9.794661190965094e-06, 'epoch': 0.08}
+{'loss': 1.8712, 'grad_norm': 2.1867644786834717, 'learning_rate': 9.791238877481179e-06, 'epoch': 0.08}
+{'loss': 1.8091, 'grad_norm': 2.038297176361084, 'learning_rate': 9.787816563997264e-06, 'epoch': 0.08}
+{'loss': 1.8654, 'grad_norm': 2.041032552719116, 'learning_rate': 9.784394250513348e-06, 'epoch': 0.08}
+{'eval_loss': 2.0577988624572754, 'eval_runtime': 11.5317, 'eval_samples_per_second': 52.03, 'eval_steps_per_second': 1.127, 'epoch': 0.08}
+{'loss': 1.8165, 'grad_norm': 1.9230015277862549, 'learning_rate': 9.780971937029433e-06, 'epoch': 0.08}
+{'loss': 1.8522, 'grad_norm': 2.4708755016326904, 'learning_rate': 9.777549623545518e-06, 'epoch': 0.08}
+{'loss': 1.841, 'grad_norm': 1.8017152547836304, 'learning_rate': 9.774127310061603e-06, 'epoch': 0.09}
+{'loss': 1.8306, 'grad_norm': 2.1635735034942627, 'learning_rate': 9.770704996577687e-06, 'epoch': 0.09}
+{'loss': 1.7945, 'grad_norm': 1.7945680618286133, 'learning_rate': 9.767282683093772e-06, 'epoch': 0.09}
+{'loss': 1.8414, 'grad_norm': 2.332819938659668, 'learning_rate': 9.763860369609857e-06, 'epoch': 0.09}
+{'loss': 1.8025, 'grad_norm': 1.7683643102645874, 'learning_rate': 9.76043805612594e-06, 'epoch': 0.09}
+{'loss': 1.8055, 'grad_norm': 2.1792047023773193, 'learning_rate': 9.757015742642026e-06, 'epoch': 0.09}
+{'loss': 1.8178, 'grad_norm': 1.839830994606018, 'learning_rate': 9.753593429158111e-06, 'epoch': 0.09}
+{'loss': 1.7639, 'grad_norm': 2.089848756790161, 'learning_rate': 9.750171115674196e-06, 'epoch': 0.09}
+{'eval_loss': 2.0276927947998047, 'eval_runtime': 11.5348, 'eval_samples_per_second': 52.016, 'eval_steps_per_second': 1.127, 'epoch': 0.09}
+{'loss': 1.791, 'grad_norm': 2.0795984268188477, 'learning_rate': 9.746748802190282e-06, 'epoch': 0.09}
+{'loss': 1.8262, 'grad_norm': 1.8953250646591187, 'learning_rate': 9.743326488706367e-06, 'epoch': 0.09}
+{'loss': 1.8132, 'grad_norm': 2.0392539501190186, 'learning_rate': 9.73990417522245e-06, 'epoch': 0.1}
+{'loss': 1.7936, 'grad_norm': 1.7300399541854858, 'learning_rate': 9.736481861738536e-06, 'epoch': 0.1}
+{'loss': 1.8533, 'grad_norm': 2.0103321075439453, 'learning_rate': 9.733059548254621e-06, 'epoch': 0.1}
+{'loss': 1.7396, 'grad_norm': 2.0879769325256348, 'learning_rate': 9.729637234770706e-06, 'epoch': 0.1}
+{'loss': 1.8141, 'grad_norm': 2.376507520675659, 'learning_rate': 9.726214921286791e-06, 'epoch': 0.1}
+{'loss': 1.7261, 'grad_norm': 2.0989043712615967, 'learning_rate': 9.722792607802875e-06, 'epoch': 0.1}
+{'loss': 1.7309, 'grad_norm': 1.9840986728668213, 'learning_rate': 9.71937029431896e-06, 'epoch': 0.1}
+{'loss': 1.7521, 'grad_norm': 2.163317918777466, 'learning_rate': 9.715947980835045e-06, 'epoch': 0.1}
+{'eval_loss': 2.0125696659088135, 'eval_runtime': 11.5274, 'eval_samples_per_second': 52.05, 'eval_steps_per_second': 1.128, 'epoch': 0.1}
+{'loss': 1.7709, 'grad_norm': 1.8134108781814575, 'learning_rate': 9.71252566735113e-06, 'epoch': 0.1}
+{'loss': 1.7533, 'grad_norm': 1.881934642791748, 'learning_rate': 9.709103353867216e-06, 'epoch': 0.1}
+{'loss': 1.7249, 'grad_norm': 1.798551321029663, 'learning_rate': 9.705681040383301e-06, 'epoch': 0.11}
+{'loss': 1.7371, 'grad_norm': 1.8665673732757568, 'learning_rate': 9.702258726899384e-06, 'epoch': 0.11}
+{'loss': 1.8086, 'grad_norm': 2.1302592754364014, 'learning_rate': 9.69883641341547e-06, 'epoch': 0.11}
+{'loss': 1.789, 'grad_norm': 2.002692699432373, 'learning_rate': 9.695414099931555e-06, 'epoch': 0.11}
+{'loss': 1.7918, 'grad_norm': 1.9792057275772095, 'learning_rate': 9.69199178644764e-06, 'epoch': 0.11}
+{'loss': 1.7661, 'grad_norm': 2.0386698246002197, 'learning_rate': 9.688569472963725e-06, 'epoch': 0.11}
+{'loss': 1.682, 'grad_norm': 1.917017936706543, 'learning_rate': 9.685147159479809e-06, 'epoch': 0.11}
+{'loss': 1.7335, 'grad_norm': 1.9866483211517334, 'learning_rate': 9.681724845995894e-06, 'epoch': 0.11}
+{'eval_loss': 2.0190446376800537, 'eval_runtime': 11.5495, 'eval_samples_per_second': 51.95, 'eval_steps_per_second': 1.126, 'epoch': 0.11}
+{'loss': 1.671, 'grad_norm': 1.9181839227676392, 'learning_rate': 9.678302532511978e-06, 'epoch': 0.11}
+{'loss': 1.7552, 'grad_norm': 2.226654291152954, 'learning_rate': 9.674880219028063e-06, 'epoch': 0.11}
+{'loss': 1.7279, 'grad_norm': 1.7531609535217285, 'learning_rate': 9.671457905544148e-06, 'epoch': 0.12}
+{'loss': 1.745, 'grad_norm': 2.338890790939331, 'learning_rate': 9.668035592060233e-06, 'epoch': 0.12}
+{'loss': 1.6892, 'grad_norm': 1.8851141929626465, 'learning_rate': 9.664613278576319e-06, 'epoch': 0.12}
+{'loss': 1.8074, 'grad_norm': 2.0738070011138916, 'learning_rate': 9.661190965092404e-06, 'epoch': 0.12}
+{'loss': 1.7333, 'grad_norm': 2.122523307800293, 'learning_rate': 9.657768651608487e-06, 'epoch': 0.12}
+{'loss': 1.7498, 'grad_norm': 2.1073336601257324, 'learning_rate': 9.654346338124573e-06, 'epoch': 0.12}
+{'loss': 1.7532, 'grad_norm': 1.9685720205307007, 'learning_rate': 9.650924024640658e-06, 'epoch': 0.12}
+{'loss': 1.8276, 'grad_norm': 2.3537447452545166, 'learning_rate': 9.647501711156743e-06, 'epoch': 0.12}
+{'eval_loss': 1.9996358156204224, 'eval_runtime': 11.5363, 'eval_samples_per_second': 52.01, 'eval_steps_per_second': 1.127, 'epoch': 0.12}
+{'loss': 1.751, 'grad_norm': 1.8049235343933105, 'learning_rate': 9.644079397672828e-06, 'epoch': 0.12}
+{'loss': 1.7313, 'grad_norm': 2.3027150630950928, 'learning_rate': 9.640657084188912e-06, 'epoch': 0.13}
+{'loss': 1.6882, 'grad_norm': 1.947838306427002, 'learning_rate': 9.637234770704997e-06, 'epoch': 0.13}
+{'loss': 1.7206, 'grad_norm': 2.6821374893188477, 'learning_rate': 9.633812457221082e-06, 'epoch': 0.13}
+{'loss': 1.6704, 'grad_norm': 1.7744842767715454, 'learning_rate': 9.630390143737167e-06, 'epoch': 0.13}
+{'loss': 1.7371, 'grad_norm': 2.0912325382232666, 'learning_rate': 9.626967830253253e-06, 'epoch': 0.13}
+{'loss': 1.664, 'grad_norm': 1.9026362895965576, 'learning_rate': 9.623545516769338e-06, 'epoch': 0.13}
+{'loss': 1.7097, 'grad_norm': 2.1378369331359863, 'learning_rate': 9.620123203285421e-06, 'epoch': 0.13}
+{'loss': 1.765, 'grad_norm': 2.0164642333984375, 'learning_rate': 9.616700889801507e-06, 'epoch': 0.13}
+{'loss': 1.7683, 'grad_norm': 2.050016164779663, 'learning_rate': 9.613278576317592e-06, 'epoch': 0.13}
+{'eval_loss': 1.983323335647583, 'eval_runtime': 11.5253, 'eval_samples_per_second': 52.059, 'eval_steps_per_second': 1.128, 'epoch': 0.13}
+{'loss': 1.6955, 'grad_norm': 2.141329288482666, 'learning_rate': 9.609856262833677e-06, 'epoch': 0.13}
+{'loss': 1.7588, 'grad_norm': 1.8916280269622803, 'learning_rate': 9.606433949349762e-06, 'epoch': 0.14}
+{'loss': 1.726, 'grad_norm': 1.9714467525482178, 'learning_rate': 9.603011635865846e-06, 'epoch': 0.14}
+{'loss': 1.6793, 'grad_norm': 1.8799078464508057, 'learning_rate': 9.599589322381931e-06, 'epoch': 0.14}
+{'loss': 1.7232, 'grad_norm': 2.0570735931396484, 'learning_rate': 9.596167008898016e-06, 'epoch': 0.14}
+{'loss': 1.7347, 'grad_norm': 1.8815388679504395, 'learning_rate': 9.592744695414101e-06, 'epoch': 0.14}
+{'loss': 1.662, 'grad_norm': 1.9568352699279785, 'learning_rate': 9.589322381930185e-06, 'epoch': 0.14}
+{'loss': 1.7318, 'grad_norm': 2.268989324569702, 'learning_rate': 9.58590006844627e-06, 'epoch': 0.14}
+{'loss': 1.6803, 'grad_norm': 1.7474064826965332, 'learning_rate': 9.582477754962355e-06, 'epoch': 0.14}
+{'loss': 1.6861, 'grad_norm': 2.113572359085083, 'learning_rate': 9.57905544147844e-06, 'epoch': 0.14}
+{'eval_loss': 1.9544686079025269, 'eval_runtime': 11.5214, 'eval_samples_per_second': 52.077, 'eval_steps_per_second': 1.128, 'epoch': 0.14}
+{'loss': 1.718, 'grad_norm': 1.9965405464172363, 'learning_rate': 9.575633127994524e-06, 'epoch': 0.14}
+{'loss': 1.6641, 'grad_norm': 1.834114909172058, 'learning_rate': 9.57221081451061e-06, 'epoch': 0.15}
+{'loss': 1.8, 'grad_norm': 2.4099228382110596, 'learning_rate': 9.568788501026695e-06, 'epoch': 0.15}
+{'loss': 1.6754, 'grad_norm': 1.9423693418502808, 'learning_rate': 9.56536618754278e-06, 'epoch': 0.15}
+{'loss': 1.664, 'grad_norm': 2.1213057041168213, 'learning_rate': 9.561943874058865e-06, 'epoch': 0.15}
+{'loss': 1.6587, 'grad_norm': 1.8991202116012573, 'learning_rate': 9.558521560574949e-06, 'epoch': 0.15}
+{'loss': 1.6953, 'grad_norm': 2.1289379596710205, 'learning_rate': 9.555099247091034e-06, 'epoch': 0.15}
+{'loss': 1.6969, 'grad_norm': 1.9584341049194336, 'learning_rate': 9.551676933607119e-06, 'epoch': 0.15}
+{'loss': 1.7837, 'grad_norm': 1.9492852687835693, 'learning_rate': 9.548254620123204e-06, 'epoch': 0.15}
+{'loss': 1.7182, 'grad_norm': 1.910580039024353, 'learning_rate': 9.54483230663929e-06, 'epoch': 0.15}
+{'eval_loss': 1.9453887939453125, 'eval_runtime': 11.5536, 'eval_samples_per_second': 51.932, 'eval_steps_per_second': 1.125, 'epoch': 0.15}
+{'loss': 1.6252, 'grad_norm': 1.8135701417922974, 'learning_rate': 9.541409993155375e-06, 'epoch': 0.16}
+{'loss': 1.658, 'grad_norm': 1.8740917444229126, 'learning_rate': 9.537987679671458e-06, 'epoch': 0.16}
+{'loss': 1.7135, 'grad_norm': 2.038465976715088, 'learning_rate': 9.534565366187543e-06, 'epoch': 0.16}
+{'loss': 1.7024, 'grad_norm': 1.9596689939498901, 'learning_rate': 9.531143052703629e-06, 'epoch': 0.16}
+{'loss': 1.65, 'grad_norm': 1.90118408203125, 'learning_rate': 9.527720739219714e-06, 'epoch': 0.16}
+{'loss': 1.6194, 'grad_norm': 1.9978408813476562, 'learning_rate': 9.524298425735799e-06, 'epoch': 0.16}
+{'loss': 1.6817, 'grad_norm': 1.9306751489639282, 'learning_rate': 9.520876112251883e-06, 'epoch': 0.16}
+{'loss': 1.6856, 'grad_norm': 1.980381727218628, 'learning_rate': 9.517453798767968e-06, 'epoch': 0.16}
+{'loss': 1.6771, 'grad_norm': 1.8978346586227417, 'learning_rate': 9.514031485284053e-06, 'epoch': 0.16}
+{'loss': 1.6888, 'grad_norm': 2.1039364337921143, 'learning_rate': 9.510609171800138e-06, 'epoch': 0.16}
+{'eval_loss': 1.9424564838409424, 'eval_runtime': 11.4167, 'eval_samples_per_second': 52.554, 'eval_steps_per_second': 1.139, 'epoch': 0.16}
+{'loss': 1.6843, 'grad_norm': 1.7545086145401, 'learning_rate': 9.507186858316223e-06, 'epoch': 0.17}
+{'loss': 1.6676, 'grad_norm': 1.781374216079712, 'learning_rate': 9.503764544832309e-06, 'epoch': 0.17}
+{'loss': 1.6338, 'grad_norm': 1.827106237411499, 'learning_rate': 9.500342231348392e-06, 'epoch': 0.17}
+{'loss': 1.6786, 'grad_norm': 1.9784914255142212, 'learning_rate': 9.496919917864477e-06, 'epoch': 0.17}
+{'loss': 1.7259, 'grad_norm': 1.8343596458435059, 'learning_rate': 9.493497604380561e-06, 'epoch': 0.17}
+{'loss': 1.7004, 'grad_norm': 1.8958734273910522, 'learning_rate': 9.490075290896646e-06, 'epoch': 0.17}
+{'loss': 1.6377, 'grad_norm': 1.925555944442749, 'learning_rate': 9.486652977412731e-06, 'epoch': 0.17}
+{'loss': 1.687, 'grad_norm': 1.8795826435089111, 'learning_rate': 9.483230663928817e-06, 'epoch': 0.17}
+{'loss': 1.7291, 'grad_norm': 1.89240562915802, 'learning_rate': 9.479808350444902e-06, 'epoch': 0.17}
+{'loss': 1.7029, 'grad_norm': 1.8946077823638916, 'learning_rate': 9.476386036960985e-06, 'epoch': 0.17}
+{'eval_loss': 1.9437302350997925, 'eval_runtime': 11.4987, 'eval_samples_per_second': 52.18, 'eval_steps_per_second': 1.131, 'epoch': 0.17}
+{'loss': 1.655, 'grad_norm': 2.0084969997406006, 'learning_rate': 9.47296372347707e-06, 'epoch': 0.18}
+{'loss': 1.6452, 'grad_norm': 1.9189505577087402, 'learning_rate': 9.469541409993156e-06, 'epoch': 0.18}
+{'loss': 1.6421, 'grad_norm': 1.9349111318588257, 'learning_rate': 9.466119096509241e-06, 'epoch': 0.18}
+{'loss': 1.5932, 'grad_norm': 2.02848482131958, 'learning_rate': 9.462696783025326e-06, 'epoch': 0.18}
+{'loss': 1.7135, 'grad_norm': 2.1285417079925537, 'learning_rate': 9.459274469541411e-06, 'epoch': 0.18}
+{'loss': 1.6861, 'grad_norm': 2.0271506309509277, 'learning_rate': 9.455852156057495e-06, 'epoch': 0.18}
+{'loss': 1.6089, 'grad_norm': 1.9525021314620972, 'learning_rate': 9.45242984257358e-06, 'epoch': 0.18}
+{'loss': 1.687, 'grad_norm': 1.9755468368530273, 'learning_rate': 9.449007529089665e-06, 'epoch': 0.18}
+{'loss': 1.5771, 'grad_norm': 1.9362246990203857, 'learning_rate': 9.44558521560575e-06, 'epoch': 0.18}
+{'loss': 1.7331, 'grad_norm': 1.9721753597259521, 'learning_rate': 9.442162902121836e-06, 'epoch': 0.18}
+{'eval_loss': 1.9382539987564087, 'eval_runtime': 11.5075, 'eval_samples_per_second': 52.14, 'eval_steps_per_second': 1.13, 'epoch': 0.18}
+{'loss': 1.7037, 'grad_norm': 2.1037437915802, 'learning_rate': 9.43874058863792e-06, 'epoch': 0.19}
+{'loss': 1.7066, 'grad_norm': 1.9612191915512085, 'learning_rate': 9.435318275154005e-06, 'epoch': 0.19}
+{'loss': 1.7168, 'grad_norm': 1.8720465898513794, 'learning_rate': 9.43189596167009e-06, 'epoch': 0.19}
+{'loss': 1.573, 'grad_norm': 1.8674277067184448, 'learning_rate': 9.428473648186175e-06, 'epoch': 0.19}
+{'loss': 1.6041, 'grad_norm': 1.7550997734069824, 'learning_rate': 9.42505133470226e-06, 'epoch': 0.19}
+{'loss': 1.6056, 'grad_norm': 1.8436589241027832, 'learning_rate': 9.421629021218345e-06, 'epoch': 0.19}
+{'loss': 1.6228, 'grad_norm': 2.1146023273468018, 'learning_rate': 9.418206707734429e-06, 'epoch': 0.19}
+{'loss': 1.6785, 'grad_norm': 1.945655107498169, 'learning_rate': 9.414784394250514e-06, 'epoch': 0.19}
+{'loss': 1.6628, 'grad_norm': 2.03094220161438, 'learning_rate': 9.4113620807666e-06, 'epoch': 0.19}
+{'loss': 1.5978, 'grad_norm': 1.8089008331298828, 'learning_rate': 9.407939767282683e-06, 'epoch': 0.2}
+{'eval_loss': 1.923292636871338, 'eval_runtime': 11.4945, 'eval_samples_per_second': 52.199, 'eval_steps_per_second': 1.131, 'epoch': 0.2}
+{'loss': 1.6742, 'grad_norm': 1.8527060747146606, 'learning_rate': 9.404517453798768e-06, 'epoch': 0.2}
+{'loss': 1.6324, 'grad_norm': 1.9459693431854248, 'learning_rate': 9.401095140314853e-06, 'epoch': 0.2}
+{'loss': 1.5955, 'grad_norm': 1.8584229946136475, 'learning_rate': 9.397672826830939e-06, 'epoch': 0.2}
+{'loss': 1.6556, 'grad_norm': 2.1260910034179688, 'learning_rate': 9.394250513347022e-06, 'epoch': 0.2}
+{'loss': 1.6106, 'grad_norm': 1.9477425813674927, 'learning_rate': 9.390828199863107e-06, 'epoch': 0.2}
+{'loss': 1.6207, 'grad_norm': 2.1237452030181885, 'learning_rate': 9.387405886379193e-06, 'epoch': 0.2}
+{'loss': 1.62, 'grad_norm': 1.908958911895752, 'learning_rate': 9.383983572895278e-06, 'epoch': 0.2}
+{'loss': 1.579, 'grad_norm': 1.897413730621338, 'learning_rate': 9.380561259411363e-06, 'epoch': 0.2}
+{'loss': 1.7264, 'grad_norm': 1.982938289642334, 'learning_rate': 9.377138945927448e-06, 'epoch': 0.2}
+{'loss': 1.5912, 'grad_norm': 1.8880170583724976, 'learning_rate': 9.373716632443532e-06, 'epoch': 0.21}
+{'eval_loss': 1.940456748008728, 'eval_runtime': 11.51, 'eval_samples_per_second': 52.129, 'eval_steps_per_second': 1.129, 'epoch': 0.21}
+{'loss': 1.6049, 'grad_norm': 1.9078369140625, 'learning_rate': 9.370294318959617e-06, 'epoch': 0.21}

wandb/run-20241030_222932-sh4rlbgu/files/requirements.txt

@@ -0,0 +1,147 @@
+funcsigs==1.0.2
+sentry-sdk==2.17.0
+multiprocess==0.70.16
+numpy==1.26.2
+pluralizer==1.2.0
+debugpy==1.6.7
+nvidia-cudnn-cu11==8.5.0.96
+deepspeed==0.15.2
+data==0.4
+pandas==2.1.3
+tomli==2.0.1
+charset-normalizer==3.3.2
+attrs==24.2.0
+aiosignal==1.3.1
+fsspec==2023.10.0
+nvidia-cusparse-cu11==11.7.4.91
+zipp==3.12.0
+mypy-extensions==1.0.0
+datasets==3.0.1
+joblib==1.3.2
+hjson==3.1.0
+traitlets==5.7.1
+stack-data==0.6.0
+transformers==4.45.1
+sympy==1.11.1
+Pygments==2.15.0
+docker-pycreds==0.4.0
+dill==0.3.8
+wheel==0.44.0
+prompt-toolkit==3.0.30
+parso==0.8.3
+ipykernel==6.23.1
+pyarrow==17.0.0
+certifi==2023.11.17
+nvidia-cufft-cu11==10.9.0.58
+six==1.16.0
+pydantic==2.9.2
+click==8.1.7
+nest-asyncio==1.5.6
+gmpy2==2.1.0
+matplotlib==3.8.2
+scipy==1.11.4
+typing_extensions==4.12.2
+statsmodels==0.14.0
+huggingface-hub==0.25.0
+frozenlist==1.4.1
+gpustat==1.1.1
+nvidia-nvtx-cu11==11.7.91
+safetensors==0.4.5
+stanza==1.9.2
+decorator==5.1.1
+seaborn==0.13.0
+sentencepiece==0.2.0
+PyYAML==6.0.1
+black==24.8.0
+protobuf==4.25.1
+pickleshare==0.7.5
+peft==0.13.0
+triton==2.0.0
+nvidia-cuda-runtime-cu11==11.7.99
+Jinja2==3.1.2
+nvidia-cusolver-cu11==11.4.0.1
+executing==1.2.0
+jupyter_client==8.1.0
+pluggy==1.3.0
+cmake==3.30.3
+pytz==2023.3.post1
+aiohappyeyeballs==2.4.2
+kiwisolver==1.4.5
+py-cpuinfo==9.0.0
+Pillow==10.1.0
+ptyprocess==0.7.0
+importlib_resources==6.4.5
+GitPython==3.1.43
+importlib-metadata==6.0.0
+iniconfig==2.0.0
+scikit-learn==1.3.2
+exceptiongroup==1.1.0
+networkx==2.8.6
+accelerate==1.0.0
+nltk==3.8.1
+shutilwhich==1.1.0
+fonttools==4.45.1
+future==0.18.3
+aiohttp==3.10.6
+wcwidth==0.2.5
+idna==3.6
+filelock==3.12.2
+pathspec==0.12.1
+jupyter_core==5.1.0
+lit==18.1.8
+nvidia-curand-cu11==10.2.10.91
+nvidia-cublas-cu11==11.10.3.66
+nvidia-ml-py==12.560.30
+msgpack==1.1.0
+python-dateutil==2.8.2
+blessed==1.20.0
+packaging==23.0
+gitdb==4.0.11
+yarl==1.13.0
+emoji==2.8.0
+tzdata==2023.3
+cycler==0.12.1
+tornado==6.2
+backcall==0.2.0
+plotnine==0.12.4
+ninja==1.11.1.1
+latex==0.7.0
+wandb==0.18.5
+setproctitle==1.3.3
+threadpoolctl==3.2.0
+requests==2.32.3
+pyparsing==3.1.1
+smmap==5.0.1
+pyzmq==23.0.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+matplotlib-inline==0.1.6
+latexcodec==1.0.0
+ipython==8.0.0
+patsy==0.5.3
+contourpy==1.2.0
+multidict==6.1.0
+mizani==0.9.3
+urllib3==2.1.0
+tokenizers==0.20.0
+MarkupSafe==2.1.2
+pip==24.2
+pexpect==4.8.0
+tqdm==4.66.5
+jedi==0.18.2
+pydantic_core==2.23.4
+tempdir==0.7.1
+mpmath==1.2.1
+setuptools==72.1.0
+pytest==7.4.3
+pure-eval==0.2.2
+psutil==5.9.1
+comm==0.1.2
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+regex==2023.10.3
+platformdirs==2.5.2
+asttokens==2.2.1
+torch==2.0.0
+nvidia-nccl-cu11==2.14.3
+xxhash==3.5.0

wandb/run-20241030_222932-sh4rlbgu/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-10-31T02:29:32.388382Z",
+  "args":  [
+    "--perturbation",
+    "reverse_control",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "3",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1710969503744"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241030_222932-sh4rlbgu/logs/debug-internal.log

@@ -0,0 +1,8 @@
+{"time":"2024-10-30T22:29:32.391426851-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-10-30T22:29:32.391443181-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_222932-sh4rlbgu/logs/debug-core.log"}
+{"time":"2024-10-30T22:29:32.49880507-04:00","level":"INFO","msg":"created new stream","id":"sh4rlbgu"}
+{"time":"2024-10-30T22:29:32.498872511-04:00","level":"INFO","msg":"stream: started","id":"sh4rlbgu"}
+{"time":"2024-10-30T22:29:32.498906531-04:00","level":"INFO","msg":"sender: started","stream_id":"sh4rlbgu"}
+{"time":"2024-10-30T22:29:32.498890911-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"sh4rlbgu"}}
+{"time":"2024-10-30T22:29:32.498912861-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"sh4rlbgu"}}
+{"time":"2024-10-30T22:29:32.680010526-04:00","level":"INFO","msg":"Starting system monitor"}

wandb/run-20241030_222932-sh4rlbgu/logs/debug.log

@@ -0,0 +1,29 @@
+2024-10-30 22:29:32,386 INFO    MainThread:447695 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-10-30 22:29:32,386 INFO    MainThread:447695 [wandb_setup.py:_flush():79] Configure stats pid to 447695
+2024-10-30 22:29:32,386 INFO    MainThread:447695 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-10-30 22:29:32,386 INFO    MainThread:447695 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-10-30 22:29:32,386 INFO    MainThread:447695 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-30 22:29:32,386 INFO    MainThread:447695 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-30 22:29:32,386 INFO    MainThread:447695 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-10-30 22:29:32,386 INFO    MainThread:447695 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-30 22:29:32,386 INFO    MainThread:447695 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_222932-sh4rlbgu/logs/debug.log
+2024-10-30 22:29:32,386 INFO    MainThread:447695 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_222932-sh4rlbgu/logs/debug-internal.log
+2024-10-30 22:29:32,387 INFO    MainThread:447695 [wandb_init.py:init():621] calling init triggers
+2024-10-30 22:29:32,387 INFO    MainThread:447695 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-10-30 22:29:32,387 INFO    MainThread:447695 [wandb_init.py:init():671] starting backend
+2024-10-30 22:29:32,387 INFO    MainThread:447695 [wandb_init.py:init():675] sending inform_init request
+2024-10-30 22:29:32,388 INFO    MainThread:447695 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-30 22:29:32,388 INFO    MainThread:447695 [wandb_init.py:init():688] backend started and connected
+2024-10-30 22:29:32,391 INFO    MainThread:447695 [wandb_init.py:init():783] updated telemetry
+2024-10-30 22:29:32,423 INFO    MainThread:447695 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-10-30 22:29:32,676 INFO    MainThread:447695 [wandb_init.py:init():867] starting run threads in backend
+2024-10-30 22:29:32,829 INFO    MainThread:447695 [wandb_run.py:_console_start():2463] atexit reg
+2024-10-30 22:29:32,829 INFO    MainThread:447695 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-10-30 22:29:32,829 INFO    MainThread:447695 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-10-30 22:29:32,829 INFO    MainThread:447695 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-10-30 22:29:32,831 INFO    MainThread:447695 [wandb_init.py:init():911] run started, returning control to user process
+2024-10-30 22:29:32,832 INFO    MainThread:447695 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_control', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0}
+2024-10-30 22:30:33,674 INFO    MainThread:447695 [wandb_run.py:_config_callback():1390] config_cb None None {'vocab_size': 128256, 'max_position_embeddings': 131072, 'hidden_size': 3072, 'intermediate_size': 8192, 'num_hidden_layers': 28, 'num_attention_heads': 24, 'num_key_value_heads': 8, 'hidden_act': 'silu', 'initializer_range': 0.02, 'rms_norm_eps': 1e-05, 'pretraining_tp': 1, 'use_cache': True, 'rope_theta': 500000.0, 'rope_scaling': {'factor': 32.0, 'high_freq_factor': 4.0, 'low_freq_factor': 1.0, 'original_max_position_embeddings': 8192, 'rope_type': 'llama3'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 128, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'bfloat16', 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': True, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['LlamaForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 128000, 'pad_token_id': None, 'eos_token_id': 128001, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'meta-llama/Llama-3.2-3B', 'transformers_version': '4.45.1', 'model_type': 'llama', 'output_dir': './checkpoints/Llama-3.2-3B/babylm_reverse_control_10M_seed0/runs', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': True, 'do_predict': False, 'eval_strategy': 'steps', 'prediction_loss_only': False, 'per_device_train_batch_size': 3, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'eval_delay': 0, 'torch_empty_cache_steps': None, 'learning_rate': 1e-05, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 3, 'max_steps': -1, 'lr_scheduler_type': 'linear', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.0, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': './logs', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 1, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 150, 'save_total_limit': None, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 0, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': False, 'fp16': True, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 10, 'dataloader_num_workers': 0, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': './checkpoints/Llama-3.2-3B/babylm_reverse_control_10M_seed0/runs', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': 'deepspeed_config/train_dp_config.json', 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'evaluation_strategy': 'steps', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False, 'use_liger_kernel': False, 'eval_use_gather_object': False}
+2024-10-30 22:30:33,677 INFO    MainThread:447695 [wandb_config.py:__setitem__():154] config set model/num_parameters = 3212749824 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x7f4e59f99ca0>>
+2024-10-30 22:30:33,677 INFO    MainThread:447695 [wandb_run.py:_config_callback():1390] config_cb model/num_parameters 3212749824 None

wandb/run-20241030_225833-8n2ypqsf/files/requirements.txt

@@ -0,0 +1,147 @@
+funcsigs==1.0.2
+sentry-sdk==2.17.0
+multiprocess==0.70.16
+numpy==1.26.2
+pluralizer==1.2.0
+debugpy==1.6.7
+nvidia-cudnn-cu11==8.5.0.96
+deepspeed==0.15.2
+data==0.4
+pandas==2.1.3
+tomli==2.0.1
+charset-normalizer==3.3.2
+attrs==24.2.0
+aiosignal==1.3.1
+fsspec==2023.10.0
+nvidia-cusparse-cu11==11.7.4.91
+zipp==3.12.0
+mypy-extensions==1.0.0
+datasets==3.0.1
+joblib==1.3.2
+hjson==3.1.0
+traitlets==5.7.1
+stack-data==0.6.0
+transformers==4.45.1
+sympy==1.11.1
+Pygments==2.15.0
+docker-pycreds==0.4.0
+dill==0.3.8
+wheel==0.44.0
+prompt-toolkit==3.0.30
+parso==0.8.3
+ipykernel==6.23.1
+pyarrow==17.0.0
+certifi==2023.11.17
+nvidia-cufft-cu11==10.9.0.58
+six==1.16.0
+pydantic==2.9.2
+click==8.1.7
+nest-asyncio==1.5.6
+gmpy2==2.1.0
+matplotlib==3.8.2
+scipy==1.11.4
+typing_extensions==4.12.2
+statsmodels==0.14.0
+huggingface-hub==0.25.0
+frozenlist==1.4.1
+gpustat==1.1.1
+nvidia-nvtx-cu11==11.7.91
+safetensors==0.4.5
+stanza==1.9.2
+decorator==5.1.1
+seaborn==0.13.0
+sentencepiece==0.2.0
+PyYAML==6.0.1
+black==24.8.0
+protobuf==4.25.1
+pickleshare==0.7.5
+peft==0.13.0
+triton==2.0.0
+nvidia-cuda-runtime-cu11==11.7.99
+Jinja2==3.1.2
+nvidia-cusolver-cu11==11.4.0.1
+executing==1.2.0
+jupyter_client==8.1.0
+pluggy==1.3.0
+cmake==3.30.3
+pytz==2023.3.post1
+aiohappyeyeballs==2.4.2
+kiwisolver==1.4.5
+py-cpuinfo==9.0.0
+Pillow==10.1.0
+ptyprocess==0.7.0
+importlib_resources==6.4.5
+GitPython==3.1.43
+importlib-metadata==6.0.0
+iniconfig==2.0.0
+scikit-learn==1.3.2
+exceptiongroup==1.1.0
+networkx==2.8.6
+accelerate==1.0.0
+nltk==3.8.1
+shutilwhich==1.1.0
+fonttools==4.45.1
+future==0.18.3
+aiohttp==3.10.6
+wcwidth==0.2.5
+idna==3.6
+filelock==3.12.2
+pathspec==0.12.1
+jupyter_core==5.1.0
+lit==18.1.8
+nvidia-curand-cu11==10.2.10.91
+nvidia-cublas-cu11==11.10.3.66
+nvidia-ml-py==12.560.30
+msgpack==1.1.0
+python-dateutil==2.8.2
+blessed==1.20.0
+packaging==23.0
+gitdb==4.0.11
+yarl==1.13.0
+emoji==2.8.0
+tzdata==2023.3
+cycler==0.12.1
+tornado==6.2
+backcall==0.2.0
+plotnine==0.12.4
+ninja==1.11.1.1
+latex==0.7.0
+wandb==0.18.5
+setproctitle==1.3.3
+threadpoolctl==3.2.0
+requests==2.32.3
+pyparsing==3.1.1
+smmap==5.0.1
+pyzmq==23.0.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+matplotlib-inline==0.1.6
+latexcodec==1.0.0
+ipython==8.0.0
+patsy==0.5.3
+contourpy==1.2.0
+multidict==6.1.0
+mizani==0.9.3
+urllib3==2.1.0
+tokenizers==0.20.0
+MarkupSafe==2.1.2
+pip==24.2
+pexpect==4.8.0
+tqdm==4.66.5
+jedi==0.18.2
+pydantic_core==2.23.4
+tempdir==0.7.1
+mpmath==1.2.1
+setuptools==72.1.0
+pytest==7.4.3
+pure-eval==0.2.2
+psutil==5.9.1
+comm==0.1.2
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+regex==2023.10.3
+platformdirs==2.5.2
+asttokens==2.2.1
+torch==2.0.0
+nvidia-nccl-cu11==2.14.3
+xxhash==3.5.0

wandb/run-20241030_225833-8n2ypqsf/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-10-31T02:58:33.877727Z",
+  "args":  [
+    "--perturbation",
+    "reverse_full",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "3",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1710970511360"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241030_225833-8n2ypqsf/logs/debug-internal.log

@@ -0,0 +1,8 @@
+{"time":"2024-10-30T22:58:33.879612833-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-10-30T22:58:33.879627353-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_225833-8n2ypqsf/logs/debug-core.log"}
+{"time":"2024-10-30T22:58:33.985315098-04:00","level":"INFO","msg":"created new stream","id":"8n2ypqsf"}
+{"time":"2024-10-30T22:58:33.985352588-04:00","level":"INFO","msg":"stream: started","id":"8n2ypqsf"}
+{"time":"2024-10-30T22:58:33.985386238-04:00","level":"INFO","msg":"sender: started","stream_id":"8n2ypqsf"}
+{"time":"2024-10-30T22:58:33.985391678-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"8n2ypqsf"}}
+{"time":"2024-10-30T22:58:33.985382408-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"8n2ypqsf"}}
+{"time":"2024-10-30T22:58:34.313843425-04:00","level":"INFO","msg":"Starting system monitor"}

wandb/run-20241030_225833-8n2ypqsf/logs/debug.log

@@ -0,0 +1,26 @@
+2024-10-30 22:58:33,875 INFO    MainThread:451912 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-10-30 22:58:33,875 INFO    MainThread:451912 [wandb_setup.py:_flush():79] Configure stats pid to 451912
+2024-10-30 22:58:33,875 INFO    MainThread:451912 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-10-30 22:58:33,875 INFO    MainThread:451912 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-10-30 22:58:33,875 INFO    MainThread:451912 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-30 22:58:33,875 INFO    MainThread:451912 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-30 22:58:33,875 INFO    MainThread:451912 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-10-30 22:58:33,875 INFO    MainThread:451912 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-30 22:58:33,875 INFO    MainThread:451912 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_225833-8n2ypqsf/logs/debug.log
+2024-10-30 22:58:33,876 INFO    MainThread:451912 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_225833-8n2ypqsf/logs/debug-internal.log
+2024-10-30 22:58:33,876 INFO    MainThread:451912 [wandb_init.py:init():621] calling init triggers
+2024-10-30 22:58:33,876 INFO    MainThread:451912 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-10-30 22:58:33,876 INFO    MainThread:451912 [wandb_init.py:init():671] starting backend
+2024-10-30 22:58:33,876 INFO    MainThread:451912 [wandb_init.py:init():675] sending inform_init request
+2024-10-30 22:58:33,877 INFO    MainThread:451912 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-30 22:58:33,877 INFO    MainThread:451912 [wandb_init.py:init():688] backend started and connected
+2024-10-30 22:58:33,880 INFO    MainThread:451912 [wandb_init.py:init():783] updated telemetry
+2024-10-30 22:58:33,918 INFO    MainThread:451912 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-10-30 22:58:34,310 INFO    MainThread:451912 [wandb_init.py:init():867] starting run threads in backend
+2024-10-30 22:58:34,438 INFO    MainThread:451912 [wandb_run.py:_console_start():2463] atexit reg
+2024-10-30 22:58:34,439 INFO    MainThread:451912 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-10-30 22:58:34,439 INFO    MainThread:451912 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-10-30 22:58:34,439 INFO    MainThread:451912 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-10-30 22:58:34,441 INFO    MainThread:451912 [wandb_init.py:init():911] run started, returning control to user process
+2024-10-30 22:58:34,442 INFO    MainThread:451912 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_full', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0}

wandb/run-20241030_231835-tt0m4qwl/logs/debug-internal.log

@@ -0,0 +1,8 @@
+{"time":"2024-10-30T23:18:35.224577864-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-10-30T23:18:35.224590524-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_231835-tt0m4qwl/logs/debug-core.log"}
+{"time":"2024-10-30T23:18:35.331278766-04:00","level":"INFO","msg":"created new stream","id":"tt0m4qwl"}
+{"time":"2024-10-30T23:18:35.331306166-04:00","level":"INFO","msg":"stream: started","id":"tt0m4qwl"}
+{"time":"2024-10-30T23:18:35.331349607-04:00","level":"INFO","msg":"sender: started","stream_id":"tt0m4qwl"}
+{"time":"2024-10-30T23:18:35.331323777-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"tt0m4qwl"}}
+{"time":"2024-10-30T23:18:35.331352517-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"tt0m4qwl"}}
+{"time":"2024-10-30T23:18:35.53234522-04:00","level":"INFO","msg":"Starting system monitor"}

wandb/run-20241031_000839-ctnl5lvc/files/output.log

@@ -0,0 +1,13 @@
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:18<00:00,  9.32s/it]
+tokenized_valid: Dataset({
+    features: ['input_ids', 'attention_mask'],
+    num_rows: 600
+})
+/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
+  warnings.warn(
+[2024-10-31 00:09:00,194] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2024-10-31 00:09:09,716] [INFO] [comm.py:652:init_distributed] cdb=None
+Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination
+Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...
+Loading extension module cpu_adam...
+Time to load cpu_adam op: 5.350019693374634 seconds

wandb/run-20241031_000839-ctnl5lvc/files/requirements.txt

@@ -0,0 +1,147 @@
+funcsigs==1.0.2
+sentry-sdk==2.17.0
+multiprocess==0.70.16
+numpy==1.26.2
+pluralizer==1.2.0
+debugpy==1.6.7
+nvidia-cudnn-cu11==8.5.0.96
+deepspeed==0.15.2
+data==0.4
+pandas==2.1.3
+tomli==2.0.1
+charset-normalizer==3.3.2
+attrs==24.2.0
+aiosignal==1.3.1
+fsspec==2023.10.0
+nvidia-cusparse-cu11==11.7.4.91
+zipp==3.12.0
+mypy-extensions==1.0.0
+datasets==3.0.1
+joblib==1.3.2
+hjson==3.1.0
+traitlets==5.7.1
+stack-data==0.6.0
+transformers==4.45.1
+sympy==1.11.1
+Pygments==2.15.0
+docker-pycreds==0.4.0
+dill==0.3.8
+wheel==0.44.0
+prompt-toolkit==3.0.30
+parso==0.8.3
+ipykernel==6.23.1
+pyarrow==17.0.0
+certifi==2023.11.17
+nvidia-cufft-cu11==10.9.0.58
+six==1.16.0
+pydantic==2.9.2
+click==8.1.7
+nest-asyncio==1.5.6
+gmpy2==2.1.0
+matplotlib==3.8.2
+scipy==1.11.4
+typing_extensions==4.12.2
+statsmodels==0.14.0
+huggingface-hub==0.25.0
+frozenlist==1.4.1
+gpustat==1.1.1
+nvidia-nvtx-cu11==11.7.91
+safetensors==0.4.5
+stanza==1.9.2
+decorator==5.1.1
+seaborn==0.13.0
+sentencepiece==0.2.0
+PyYAML==6.0.1
+black==24.8.0
+protobuf==4.25.1
+pickleshare==0.7.5
+peft==0.13.0
+triton==2.0.0
+nvidia-cuda-runtime-cu11==11.7.99
+Jinja2==3.1.2
+nvidia-cusolver-cu11==11.4.0.1
+executing==1.2.0
+jupyter_client==8.1.0
+pluggy==1.3.0
+cmake==3.30.3
+pytz==2023.3.post1
+aiohappyeyeballs==2.4.2
+kiwisolver==1.4.5
+py-cpuinfo==9.0.0
+Pillow==10.1.0
+ptyprocess==0.7.0
+importlib_resources==6.4.5
+GitPython==3.1.43
+importlib-metadata==6.0.0
+iniconfig==2.0.0
+scikit-learn==1.3.2
+exceptiongroup==1.1.0
+networkx==2.8.6
+accelerate==1.0.0
+nltk==3.8.1
+shutilwhich==1.1.0
+fonttools==4.45.1
+future==0.18.3
+aiohttp==3.10.6
+wcwidth==0.2.5
+idna==3.6
+filelock==3.12.2
+pathspec==0.12.1
+jupyter_core==5.1.0
+lit==18.1.8
+nvidia-curand-cu11==10.2.10.91
+nvidia-cublas-cu11==11.10.3.66
+nvidia-ml-py==12.560.30
+msgpack==1.1.0
+python-dateutil==2.8.2
+blessed==1.20.0
+packaging==23.0
+gitdb==4.0.11
+yarl==1.13.0
+emoji==2.8.0
+tzdata==2023.3
+cycler==0.12.1
+tornado==6.2
+backcall==0.2.0
+plotnine==0.12.4
+ninja==1.11.1.1
+latex==0.7.0
+wandb==0.18.5
+setproctitle==1.3.3
+threadpoolctl==3.2.0
+requests==2.32.3
+pyparsing==3.1.1
+smmap==5.0.1
+pyzmq==23.0.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+matplotlib-inline==0.1.6
+latexcodec==1.0.0
+ipython==8.0.0
+patsy==0.5.3
+contourpy==1.2.0
+multidict==6.1.0
+mizani==0.9.3
+urllib3==2.1.0
+tokenizers==0.20.0
+MarkupSafe==2.1.2
+pip==24.2
+pexpect==4.8.0
+tqdm==4.66.5
+jedi==0.18.2
+pydantic_core==2.23.4
+tempdir==0.7.1
+mpmath==1.2.1
+setuptools==72.1.0
+pytest==7.4.3
+pure-eval==0.2.2
+psutil==5.9.1
+comm==0.1.2
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+regex==2023.10.3
+platformdirs==2.5.2
+asttokens==2.2.1
+torch==2.0.0
+nvidia-nccl-cu11==2.14.3
+xxhash==3.5.0

wandb/run-20241031_000839-ctnl5lvc/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-10-31T04:08:39.124581Z",
+  "args":  [
+    "--perturbation",
+    "reverse_full",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "6",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1727270539264"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241031_000839-ctnl5lvc/logs/debug-internal.log

@@ -0,0 +1,8 @@
+{"time":"2024-10-31T00:08:39.126579154-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-10-31T00:08:39.126592314-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_000839-ctnl5lvc/logs/debug-core.log"}
+{"time":"2024-10-31T00:08:39.232175065-04:00","level":"INFO","msg":"created new stream","id":"ctnl5lvc"}
+{"time":"2024-10-31T00:08:39.232207225-04:00","level":"INFO","msg":"stream: started","id":"ctnl5lvc"}
+{"time":"2024-10-31T00:08:39.232261406-04:00","level":"INFO","msg":"sender: started","stream_id":"ctnl5lvc"}
+{"time":"2024-10-31T00:08:39.232228805-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"ctnl5lvc"}}
+{"time":"2024-10-31T00:08:39.232264286-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"ctnl5lvc"}}
+{"time":"2024-10-31T00:08:39.441361111-04:00","level":"INFO","msg":"Starting system monitor"}

wandb/run-20241031_000839-ctnl5lvc/logs/debug.log

@@ -0,0 +1,26 @@
+2024-10-31 00:08:39,123 INFO    MainThread:477296 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-10-31 00:08:39,123 INFO    MainThread:477296 [wandb_setup.py:_flush():79] Configure stats pid to 477296
+2024-10-31 00:08:39,123 INFO    MainThread:477296 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-10-31 00:08:39,123 INFO    MainThread:477296 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-10-31 00:08:39,123 INFO    MainThread:477296 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-31 00:08:39,123 INFO    MainThread:477296 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-31 00:08:39,123 INFO    MainThread:477296 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-10-31 00:08:39,123 INFO    MainThread:477296 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-31 00:08:39,123 INFO    MainThread:477296 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_000839-ctnl5lvc/logs/debug.log
+2024-10-31 00:08:39,123 INFO    MainThread:477296 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_000839-ctnl5lvc/logs/debug-internal.log
+2024-10-31 00:08:39,123 INFO    MainThread:477296 [wandb_init.py:init():621] calling init triggers
+2024-10-31 00:08:39,123 INFO    MainThread:477296 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-10-31 00:08:39,123 INFO    MainThread:477296 [wandb_init.py:init():671] starting backend
+2024-10-31 00:08:39,123 INFO    MainThread:477296 [wandb_init.py:init():675] sending inform_init request
+2024-10-31 00:08:39,124 INFO    MainThread:477296 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-31 00:08:39,124 INFO    MainThread:477296 [wandb_init.py:init():688] backend started and connected
+2024-10-31 00:08:39,126 INFO    MainThread:477296 [wandb_init.py:init():783] updated telemetry
+2024-10-31 00:08:39,152 INFO    MainThread:477296 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-10-31 00:08:39,438 INFO    MainThread:477296 [wandb_init.py:init():867] starting run threads in backend
+2024-10-31 00:08:39,549 INFO    MainThread:477296 [wandb_run.py:_console_start():2463] atexit reg
+2024-10-31 00:08:39,549 INFO    MainThread:477296 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-10-31 00:08:39,549 INFO    MainThread:477296 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-10-31 00:08:39,549 INFO    MainThread:477296 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-10-31 00:08:39,550 INFO    MainThread:477296 [wandb_init.py:init():911] run started, returning control to user process
+2024-10-31 00:08:39,551 INFO    MainThread:477296 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_full', 'train_set': '10M', 'batch_size': 3, 'epoch': 6, 'seed': 0, 'lr': 1e-05}

wandb/run-20241031_002020-q6ot1vz6/logs/debug.log

@@ -0,0 +1,33 @@
+2024-10-31 00:20:20,448 INFO    MainThread:484460 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-10-31 00:20:20,448 INFO    MainThread:484460 [wandb_setup.py:_flush():79] Configure stats pid to 484460
+2024-10-31 00:20:20,448 INFO    MainThread:484460 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-10-31 00:20:20,448 INFO    MainThread:484460 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-10-31 00:20:20,448 INFO    MainThread:484460 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-31 00:20:20,448 INFO    MainThread:484460 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-31 00:20:20,448 INFO    MainThread:484460 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-10-31 00:20:20,448 INFO    MainThread:484460 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-31 00:20:20,448 INFO    MainThread:484460 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_002020-q6ot1vz6/logs/debug.log
+2024-10-31 00:20:20,448 INFO    MainThread:484460 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_002020-q6ot1vz6/logs/debug-internal.log
+2024-10-31 00:20:20,448 INFO    MainThread:484460 [wandb_init.py:init():621] calling init triggers
+2024-10-31 00:20:20,448 INFO    MainThread:484460 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-10-31 00:20:20,448 INFO    MainThread:484460 [wandb_init.py:init():671] starting backend
+2024-10-31 00:20:20,448 INFO    MainThread:484460 [wandb_init.py:init():675] sending inform_init request
+2024-10-31 00:20:20,449 INFO    MainThread:484460 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-31 00:20:20,449 INFO    MainThread:484460 [wandb_init.py:init():688] backend started and connected
+2024-10-31 00:20:20,452 INFO    MainThread:484460 [wandb_init.py:init():783] updated telemetry
+2024-10-31 00:20:20,476 INFO    MainThread:484460 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-10-31 00:20:21,380 INFO    MainThread:484460 [wandb_init.py:init():867] starting run threads in backend
+2024-10-31 00:20:21,487 INFO    MainThread:484460 [wandb_run.py:_console_start():2463] atexit reg
+2024-10-31 00:20:21,487 INFO    MainThread:484460 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-10-31 00:20:21,487 INFO    MainThread:484460 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-10-31 00:20:21,487 INFO    MainThread:484460 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-10-31 00:20:21,489 INFO    MainThread:484460 [wandb_init.py:init():911] run started, returning control to user process
+2024-10-31 00:20:21,489 INFO    MainThread:484460 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_full', 'train_set': '10M', 'batch_size': 3, 'epoch': 6, 'seed': 0, 'lr': 1e-05}
+2024-10-31 09:17:19,685 INFO    MainThread:484460 [wandb_run.py:_finish():2158] finishing run yaning1001-dartmouth-college/impossible_llm_reverse/q6ot1vz6
+2024-10-31 09:17:19,700 INFO    MainThread:484460 [wandb_run.py:_atexit_cleanup():2426] got exitcode: 0
+2024-10-31 09:17:19,748 INFO    MainThread:484460 [wandb_run.py:_restore():2408] restore
+2024-10-31 09:17:19,749 INFO    MainThread:484460 [wandb_run.py:_restore():2414] restore done
+2024-10-31 09:17:23,121 INFO    MainThread:484460 [wandb_run.py:_footer_history_summary_info():3975] rendering history
+2024-10-31 09:17:23,121 INFO    MainThread:484460 [wandb_run.py:_footer_history_summary_info():4007] rendering summary
+2024-10-31 09:17:23,131 INFO    MainThread:484460 [wandb_run.py:_footer_sync_info():3934] logging synced files

wandb/run-20241031_002020-u516mysu/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-10-31T04:20:20.446931Z",
+  "args":  [
+    "--perturbation",
+    "reverse_full",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "6",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1735150157824"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241031_002020-u516mysu/files/wandb-summary.json

@@ -0,0 +1 @@
+{"train/global_step":2922,"train_samples_per_second":3.284,"_runtime":32219.326770301,"train_steps_per_second":0.091,"train_loss":1.267335325887479,"train/epoch":6,"eval/samples_per_second":51.772,"train/learning_rate":6.160164271047228e-08,"total_flos":1.821755618831106e+18,"_timestamp":1.7303806396836321e+09,"_wandb":{"runtime":32219},"train/grad_norm":3.2972512245178223,"train_runtime":32012.2915,"eval/loss":1.9240258932113647,"_step":3214,"train/loss":0.9615,"eval/runtime":11.5892,"eval/steps_per_second":1.122}

wandb/run-20241031_114700-xl5jvngs/files/output.log

@@ -0,0 +1,43 @@
+Downloading shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [02:32<00:00, 76.41s/it]
+Loading checkpoint shards: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:05<00:00,  2.73s/it]
+tokenized_valid: Dataset({
+    features: ['input_ids', 'attention_mask'],
+    num_rows: 600
+})
+/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
+  warnings.warn(
+[2024-10-31 11:49:40,749] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2024-10-31 11:49:49,163] [INFO] [comm.py:652:init_distributed] cdb=None
+Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination
+Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...
+Loading extension module cpu_adam...
+Time to load cpu_adam op: 4.897958517074585 seconds
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 220, in <module>
+    trainer.train()
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/trainer.py", line 2052, in train
+    return inner_training_loop(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/trainer.py", line 2388, in _inner_training_loop
+    tr_loss_step = self.training_step(model, inputs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/trainer.py", line 3518, in training_step
+    self.accelerator.backward(loss, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/accelerate/accelerator.py", line 2238, in backward
+    self.deepspeed_engine_wrapped.backward(loss, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/accelerate/utils/deepspeed.py", line 186, in backward
+    self.engine.backward(loss, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/utils/nvtx.py", line 18, in wrapped_fn
+    ret_val = func(*args, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/runtime/engine.py", line 2020, in backward
+    self.optimizer.backward(loss, retain_graph=retain_graph)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/runtime/zero/stage_1_and_2.py", line 2063, in backward
+    self.loss_scaler.backward(loss.float(), retain_graph=retain_graph)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/runtime/fp16/loss_scaler.py", line 63, in backward
+    scaled_loss.backward(retain_graph=retain_graph)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/torch/_tensor.py", line 487, in backward
+    torch.autograd.backward(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/torch/autograd/__init__.py", line 200, in backward
+    Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
+KeyboardInterrupt
+Error in atexit._run_exitfuncs:
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/ops/transformer/inference/triton/matmul_ext.py", line 27, in is_nfs_path

wandb/run-20241031_114700-xl5jvngs/files/requirements.txt

@@ -0,0 +1,147 @@
+funcsigs==1.0.2
+sentry-sdk==2.17.0
+multiprocess==0.70.16
+numpy==1.26.2
+pluralizer==1.2.0
+debugpy==1.6.7
+nvidia-cudnn-cu11==8.5.0.96
+deepspeed==0.15.2
+data==0.4
+pandas==2.1.3
+tomli==2.0.1
+charset-normalizer==3.3.2
+attrs==24.2.0
+aiosignal==1.3.1
+fsspec==2023.10.0
+nvidia-cusparse-cu11==11.7.4.91
+zipp==3.12.0
+mypy-extensions==1.0.0
+datasets==3.0.1
+joblib==1.3.2
+hjson==3.1.0
+traitlets==5.7.1
+stack-data==0.6.0
+transformers==4.45.1
+sympy==1.11.1
+Pygments==2.15.0
+docker-pycreds==0.4.0
+dill==0.3.8
+wheel==0.44.0
+prompt-toolkit==3.0.30
+parso==0.8.3
+ipykernel==6.23.1
+pyarrow==17.0.0
+certifi==2023.11.17
+nvidia-cufft-cu11==10.9.0.58
+six==1.16.0
+pydantic==2.9.2
+click==8.1.7
+nest-asyncio==1.5.6
+gmpy2==2.1.0
+matplotlib==3.8.2
+scipy==1.11.4
+typing_extensions==4.12.2
+statsmodels==0.14.0
+huggingface-hub==0.25.0
+frozenlist==1.4.1
+gpustat==1.1.1
+nvidia-nvtx-cu11==11.7.91
+safetensors==0.4.5
+stanza==1.9.2
+decorator==5.1.1
+seaborn==0.13.0
+sentencepiece==0.2.0
+PyYAML==6.0.1
+black==24.8.0
+protobuf==4.25.1
+pickleshare==0.7.5
+peft==0.13.0
+triton==2.0.0
+nvidia-cuda-runtime-cu11==11.7.99
+Jinja2==3.1.2
+nvidia-cusolver-cu11==11.4.0.1
+executing==1.2.0
+jupyter_client==8.1.0
+pluggy==1.3.0
+cmake==3.30.3
+pytz==2023.3.post1
+aiohappyeyeballs==2.4.2
+kiwisolver==1.4.5
+py-cpuinfo==9.0.0
+Pillow==10.1.0
+ptyprocess==0.7.0
+importlib_resources==6.4.5
+GitPython==3.1.43
+importlib-metadata==6.0.0
+iniconfig==2.0.0
+scikit-learn==1.3.2
+exceptiongroup==1.1.0
+networkx==2.8.6
+accelerate==1.0.0
+nltk==3.8.1
+shutilwhich==1.1.0
+fonttools==4.45.1
+future==0.18.3
+aiohttp==3.10.6
+wcwidth==0.2.5
+idna==3.6
+filelock==3.12.2
+pathspec==0.12.1
+jupyter_core==5.1.0
+lit==18.1.8
+nvidia-curand-cu11==10.2.10.91
+nvidia-cublas-cu11==11.10.3.66
+nvidia-ml-py==12.560.30
+msgpack==1.1.0
+python-dateutil==2.8.2
+blessed==1.20.0
+packaging==23.0
+gitdb==4.0.11
+yarl==1.13.0
+emoji==2.8.0
+tzdata==2023.3
+cycler==0.12.1
+tornado==6.2
+backcall==0.2.0
+plotnine==0.12.4
+ninja==1.11.1.1
+latex==0.7.0
+wandb==0.18.5
+setproctitle==1.3.3
+threadpoolctl==3.2.0
+requests==2.32.3
+pyparsing==3.1.1
+smmap==5.0.1
+pyzmq==23.0.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+matplotlib-inline==0.1.6
+latexcodec==1.0.0
+ipython==8.0.0
+patsy==0.5.3
+contourpy==1.2.0
+multidict==6.1.0
+mizani==0.9.3
+urllib3==2.1.0
+tokenizers==0.20.0
+MarkupSafe==2.1.2
+pip==24.2
+pexpect==4.8.0
+tqdm==4.66.5
+jedi==0.18.2
+pydantic_core==2.23.4
+tempdir==0.7.1
+mpmath==1.2.1
+setuptools==72.1.0
+pytest==7.4.3
+pure-eval==0.2.2
+psutil==5.9.1
+comm==0.1.2
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+regex==2023.10.3
+platformdirs==2.5.2
+asttokens==2.2.1
+torch==2.0.0
+nvidia-nccl-cu11==2.14.3
+xxhash==3.5.0

wandb/run-20241031_114700-xl5jvngs/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-10-31T15:47:00.195057Z",
+  "args":  [
+    "--perturbation",
+    "reverse_full",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "6",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1753158594560"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241031_114700-xl5jvngs/logs/debug-internal.log

@@ -0,0 +1,8 @@
+{"time":"2024-10-31T11:47:00.198431973-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-10-31T11:47:00.198457033-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_114700-xl5jvngs/logs/debug-core.log"}
+{"time":"2024-10-31T11:47:00.311644447-04:00","level":"INFO","msg":"created new stream","id":"xl5jvngs"}
+{"time":"2024-10-31T11:47:00.311692627-04:00","level":"INFO","msg":"stream: started","id":"xl5jvngs"}
+{"time":"2024-10-31T11:47:00.311728668-04:00","level":"INFO","msg":"sender: started","stream_id":"xl5jvngs"}
+{"time":"2024-10-31T11:47:00.311711738-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"xl5jvngs"}}
+{"time":"2024-10-31T11:47:00.311729598-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"xl5jvngs"}}
+{"time":"2024-10-31T11:47:00.607247923-04:00","level":"INFO","msg":"Starting system monitor"}

wandb/run-20241031_114700-xl5jvngs/logs/debug.log

@@ -0,0 +1,26 @@
+2024-10-31 11:47:00,192 INFO    MainThread:554147 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-10-31 11:47:00,192 INFO    MainThread:554147 [wandb_setup.py:_flush():79] Configure stats pid to 554147
+2024-10-31 11:47:00,192 INFO    MainThread:554147 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-10-31 11:47:00,192 INFO    MainThread:554147 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-10-31 11:47:00,192 INFO    MainThread:554147 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-31 11:47:00,192 INFO    MainThread:554147 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-31 11:47:00,192 INFO    MainThread:554147 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-10-31 11:47:00,192 INFO    MainThread:554147 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-31 11:47:00,192 INFO    MainThread:554147 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_114700-xl5jvngs/logs/debug.log
+2024-10-31 11:47:00,193 INFO    MainThread:554147 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_114700-xl5jvngs/logs/debug-internal.log
+2024-10-31 11:47:00,193 INFO    MainThread:554147 [wandb_init.py:init():621] calling init triggers
+2024-10-31 11:47:00,193 INFO    MainThread:554147 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-10-31 11:47:00,193 INFO    MainThread:554147 [wandb_init.py:init():671] starting backend
+2024-10-31 11:47:00,193 INFO    MainThread:554147 [wandb_init.py:init():675] sending inform_init request
+2024-10-31 11:47:00,194 INFO    MainThread:554147 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-31 11:47:00,194 INFO    MainThread:554147 [wandb_init.py:init():688] backend started and connected
+2024-10-31 11:47:00,198 INFO    MainThread:554147 [wandb_init.py:init():783] updated telemetry
+2024-10-31 11:47:00,224 INFO    MainThread:554147 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-10-31 11:47:00,604 INFO    MainThread:554147 [wandb_init.py:init():867] starting run threads in backend
+2024-10-31 11:47:00,694 INFO    MainThread:554147 [wandb_run.py:_console_start():2463] atexit reg
+2024-10-31 11:47:00,695 INFO    MainThread:554147 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-10-31 11:47:00,695 INFO    MainThread:554147 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-10-31 11:47:00,695 INFO    MainThread:554147 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-10-31 11:47:00,696 INFO    MainThread:554147 [wandb_init.py:init():911] run started, returning control to user process
+2024-10-31 11:47:00,696 INFO    MainThread:554147 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_full', 'train_set': '10M', 'batch_size': 3, 'epoch': 6, 'seed': 0, 'lr': 0.0001}

wandb/run-20241101_012612-kwpxhz4q/files/config.yaml

@@ -0,0 +1,49 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.9.19
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "3":
+                - 13
+                - 23
+                - 55
+            "4": 3.9.19
+            "5": 0.18.5
+            "6": 4.45.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+batch_size:
+    value: 3
+epoch:
+    value: 6
+lr:
+    value: 5e-06
+perturbation:
+    value: shuffle_nodeterministic
+seed:
+    value: 0
+train_set:
+    value: 10M

wandb/run-20241101_012612-xxe4xsbl/files/config.yaml

@@ -0,0 +1,49 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.9.19
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "3":
+                - 13
+                - 23
+                - 55
+            "4": 3.9.19
+            "5": 0.18.5
+            "6": 4.45.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+batch_size:
+    value: 3
+epoch:
+    value: 6
+lr:
+    value: 5e-06
+perturbation:
+    value: shuffle_nodeterministic
+seed:
+    value: 0
+train_set:
+    value: 10M

wandb/run-20241101_012612-xxe4xsbl/files/output.log

@@ -0,0 +1,12 @@
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 164, in <module>
+    dataset = load_dataset('babylm_dataset_test.py', name=dataset_name, trust_remote_code=True)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/load.py", line 2074, in load_dataset
+    builder_instance = load_dataset_builder(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/load.py", line 1832, in load_dataset_builder
+    builder_instance: DatasetBuilder = builder_cls(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/builder.py", line 342, in __init__
+    self.config, self.config_id = self._create_builder_config(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/builder.py", line 569, in _create_builder_config
+    raise ValueError(
+ValueError: BuilderConfig 'babylm_shuffle_nodeterministic_10M_seed0' not found. Available: ['babylm_hop_control_10M_seed0', 'babylm_hop_tokens4_10M_seed0', 'babylm_hop_words4_10M_seed0', 'babylm_reverse_control_10M_seed0', 'babylm_reverse_partial_10M_seed0', 'babylm_reverse_full_10M_seed0', 'babylm_shuffle_control_10M_seed0', 'babylm_shuffle_nondeterministic_10M_seed0', 'babylm_shuffle_deterministic21_10M_seed0', 'babylm_shuffle_deterministic57_10M_seed0', 'babylm_shuffle_deterministic84_10M_seed0', 'babylm_shuffle_local3_10M_seed0', 'babylm_shuffle_local5_10M_seed0', 'babylm_shuffle_local10_10M_seed0', 'babylm_shuffle_even_odd_10M_seed0']

wandb/run-20241101_012612-xxe4xsbl/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-11-01T05:26:12.778667Z",
+  "args":  [
+    "--perturbation",
+    "shuffle_nodeterministic",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "6",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1753992224768"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241101_012612-xxe4xsbl/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":0}}

wandb/run-20241101_012612-xxe4xsbl/logs/debug-internal.log

@@ -0,0 +1,16 @@
+{"time":"2024-11-01T01:26:12.780492422-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-11-01T01:26:12.780503152-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_012612-xxe4xsbl/logs/debug-core.log"}
+{"time":"2024-11-01T01:26:12.886257426-04:00","level":"INFO","msg":"created new stream","id":"xxe4xsbl"}
+{"time":"2024-11-01T01:26:12.886310127-04:00","level":"INFO","msg":"stream: started","id":"xxe4xsbl"}
+{"time":"2024-11-01T01:26:12.886326247-04:00","level":"INFO","msg":"sender: started","stream_id":"xxe4xsbl"}
+{"time":"2024-11-01T01:26:12.886315187-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"xxe4xsbl"}}
+{"time":"2024-11-01T01:26:12.886356707-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"xxe4xsbl"}}
+{"time":"2024-11-01T01:26:13.097917136-04:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-11-01T01:26:13.213973052-04:00","level":"INFO","msg":"stream: closing","id":"xxe4xsbl"}
+{"time":"2024-11-01T01:26:13.214021623-04:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-11-01T01:26:13.214491436-04:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-11-01T01:26:13.970357776-04:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-11-01T01:26:14.086365941-04:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"xxe4xsbl"}}
+{"time":"2024-11-01T01:26:14.086446122-04:00","level":"INFO","msg":"sender: closed","stream_id":"xxe4xsbl"}
+{"time":"2024-11-01T01:26:14.086432742-04:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"xxe4xsbl"}}
+{"time":"2024-11-01T01:26:14.086529832-04:00","level":"INFO","msg":"stream: closed","id":"xxe4xsbl"}

wandb/run-20241101_012612-xxe4xsbl/logs/debug.log

@@ -0,0 +1,27 @@
+2024-11-01 01:26:12,776 INFO    MainThread:677634 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-11-01 01:26:12,776 INFO    MainThread:677634 [wandb_setup.py:_flush():79] Configure stats pid to 677634
+2024-11-01 01:26:12,776 INFO    MainThread:677634 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-11-01 01:26:12,776 INFO    MainThread:677634 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-11-01 01:26:12,776 INFO    MainThread:677634 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-11-01 01:26:12,776 INFO    MainThread:677634 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-11-01 01:26:12,776 INFO    MainThread:677634 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-11-01 01:26:12,776 INFO    MainThread:677634 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-11-01 01:26:12,776 INFO    MainThread:677634 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_012612-xxe4xsbl/logs/debug.log
+2024-11-01 01:26:12,776 INFO    MainThread:677634 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_012612-xxe4xsbl/logs/debug-internal.log
+2024-11-01 01:26:12,776 INFO    MainThread:677634 [wandb_init.py:init():621] calling init triggers
+2024-11-01 01:26:12,776 INFO    MainThread:677634 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-11-01 01:26:12,777 INFO    MainThread:677634 [wandb_init.py:init():671] starting backend
+2024-11-01 01:26:12,777 INFO    MainThread:677634 [wandb_init.py:init():675] sending inform_init request
+2024-11-01 01:26:12,778 INFO    MainThread:677634 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-11-01 01:26:12,778 INFO    MainThread:677634 [wandb_init.py:init():688] backend started and connected
+2024-11-01 01:26:12,781 INFO    MainThread:677634 [wandb_init.py:init():783] updated telemetry
+2024-11-01 01:26:12,810 INFO    MainThread:677634 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-11-01 01:26:13,094 INFO    MainThread:677634 [wandb_init.py:init():867] starting run threads in backend
+2024-11-01 01:26:13,202 INFO    MainThread:677634 [wandb_run.py:_console_start():2463] atexit reg
+2024-11-01 01:26:13,203 INFO    MainThread:677634 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-11-01 01:26:13,203 INFO    MainThread:677634 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-11-01 01:26:13,203 INFO    MainThread:677634 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-11-01 01:26:13,204 INFO    MainThread:677634 [wandb_init.py:init():911] run started, returning control to user process
+2024-11-01 01:26:13,205 INFO    MainThread:677634 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_nodeterministic', 'train_set': '10M', 'batch_size': 3, 'epoch': 6, 'seed': 0, 'lr': 5e-06}
+2024-11-01 01:26:13,214 WARNING MsgRouterThr:677634 [router.py:message_loop():77] message_loop has been closed

wandb/run-20241101_200517-77b12390/files/config.yaml

@@ -0,0 +1,49 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.9.19
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "3":
+                - 13
+                - 23
+                - 55
+            "4": 3.9.19
+            "5": 0.18.5
+            "6": 4.45.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+batch_size:
+    value: 3
+epoch:
+    value: 3
+lr:
+    value: 5e-06
+perturbation:
+    value: shuffle_nondeterministic
+seed:
+    value: 0
+train_set:
+    value: 10M

wandb/run-20241101_200517-77b12390/files/requirements.txt

@@ -0,0 +1,147 @@
+funcsigs==1.0.2
+sentry-sdk==2.17.0
+multiprocess==0.70.16
+numpy==1.26.2
+pluralizer==1.2.0
+debugpy==1.6.7
+nvidia-cudnn-cu11==8.5.0.96
+deepspeed==0.15.2
+data==0.4
+pandas==2.1.3
+tomli==2.0.1
+charset-normalizer==3.3.2
+attrs==24.2.0
+aiosignal==1.3.1
+fsspec==2023.10.0
+nvidia-cusparse-cu11==11.7.4.91
+zipp==3.12.0
+mypy-extensions==1.0.0
+datasets==3.0.1
+joblib==1.3.2
+hjson==3.1.0
+traitlets==5.7.1
+stack-data==0.6.0
+transformers==4.45.1
+sympy==1.11.1
+Pygments==2.15.0
+docker-pycreds==0.4.0
+dill==0.3.8
+wheel==0.44.0
+prompt-toolkit==3.0.30
+parso==0.8.3
+ipykernel==6.23.1
+pyarrow==17.0.0
+certifi==2023.11.17
+nvidia-cufft-cu11==10.9.0.58
+six==1.16.0
+pydantic==2.9.2
+click==8.1.7
+nest-asyncio==1.5.6
+gmpy2==2.1.0
+matplotlib==3.8.2
+scipy==1.11.4
+typing_extensions==4.12.2
+statsmodels==0.14.0
+huggingface-hub==0.25.0
+frozenlist==1.4.1
+gpustat==1.1.1
+nvidia-nvtx-cu11==11.7.91
+safetensors==0.4.5
+stanza==1.9.2
+decorator==5.1.1
+seaborn==0.13.0
+sentencepiece==0.2.0
+PyYAML==6.0.1
+black==24.8.0
+protobuf==4.25.1
+pickleshare==0.7.5
+peft==0.13.0
+triton==2.0.0
+nvidia-cuda-runtime-cu11==11.7.99
+Jinja2==3.1.2
+nvidia-cusolver-cu11==11.4.0.1
+executing==1.2.0
+jupyter_client==8.1.0
+pluggy==1.3.0
+cmake==3.30.3
+pytz==2023.3.post1
+aiohappyeyeballs==2.4.2
+kiwisolver==1.4.5
+py-cpuinfo==9.0.0
+Pillow==10.1.0
+ptyprocess==0.7.0
+importlib_resources==6.4.5
+GitPython==3.1.43
+importlib-metadata==6.0.0
+iniconfig==2.0.0
+scikit-learn==1.3.2
+exceptiongroup==1.1.0
+networkx==2.8.6
+accelerate==1.0.0
+nltk==3.8.1
+shutilwhich==1.1.0
+fonttools==4.45.1
+future==0.18.3
+aiohttp==3.10.6
+wcwidth==0.2.5
+idna==3.6
+filelock==3.12.2
+pathspec==0.12.1
+jupyter_core==5.1.0
+lit==18.1.8
+nvidia-curand-cu11==10.2.10.91
+nvidia-cublas-cu11==11.10.3.66
+nvidia-ml-py==12.560.30
+msgpack==1.1.0
+python-dateutil==2.8.2
+blessed==1.20.0
+packaging==23.0
+gitdb==4.0.11
+yarl==1.13.0
+emoji==2.8.0
+tzdata==2023.3
+cycler==0.12.1
+tornado==6.2
+backcall==0.2.0
+plotnine==0.12.4
+ninja==1.11.1.1
+latex==0.7.0
+wandb==0.18.5
+setproctitle==1.3.3
+threadpoolctl==3.2.0
+requests==2.32.3
+pyparsing==3.1.1
+smmap==5.0.1
+pyzmq==23.0.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+matplotlib-inline==0.1.6
+latexcodec==1.0.0
+ipython==8.0.0
+patsy==0.5.3
+contourpy==1.2.0
+multidict==6.1.0
+mizani==0.9.3
+urllib3==2.1.0
+tokenizers==0.20.0
+MarkupSafe==2.1.2
+pip==24.2
+pexpect==4.8.0
+tqdm==4.66.5
+jedi==0.18.2
+pydantic_core==2.23.4
+tempdir==0.7.1
+mpmath==1.2.1
+setuptools==72.1.0
+pytest==7.4.3
+pure-eval==0.2.2
+psutil==5.9.1
+comm==0.1.2
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+regex==2023.10.3
+platformdirs==2.5.2
+asttokens==2.2.1
+torch==2.0.0
+nvidia-nccl-cu11==2.14.3
+xxhash==3.5.0

wandb/run-20241101_201708-vqhiyzcg/files/output.log

@@ -0,0 +1,13 @@
+Loading checkpoint shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:04<00:00,  2.48s/it]
+tokenized_valid: Dataset({
+    features: ['input_ids', 'attention_mask'],
+    num_rows: 600
+})
+/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
+  warnings.warn(
+[2024-11-01 20:17:16,678] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2024-11-01 20:17:25,940] [INFO] [comm.py:652:init_distributed] cdb=None
+Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination
+Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...
+Loading extension module cpu_adam...
+Time to load cpu_adam op: 5.043281316757202 seconds

wandb/run-20241101_201708-vqhiyzcg/files/requirements.txt

@@ -0,0 +1,147 @@
+funcsigs==1.0.2
+sentry-sdk==2.17.0
+multiprocess==0.70.16
+numpy==1.26.2
+pluralizer==1.2.0
+debugpy==1.6.7
+nvidia-cudnn-cu11==8.5.0.96
+deepspeed==0.15.2
+data==0.4
+pandas==2.1.3
+tomli==2.0.1
+charset-normalizer==3.3.2
+attrs==24.2.0
+aiosignal==1.3.1
+fsspec==2023.10.0
+nvidia-cusparse-cu11==11.7.4.91
+zipp==3.12.0
+mypy-extensions==1.0.0
+datasets==3.0.1
+joblib==1.3.2
+hjson==3.1.0
+traitlets==5.7.1
+stack-data==0.6.0
+transformers==4.45.1
+sympy==1.11.1
+Pygments==2.15.0
+docker-pycreds==0.4.0
+dill==0.3.8
+wheel==0.44.0
+prompt-toolkit==3.0.30
+parso==0.8.3
+ipykernel==6.23.1
+pyarrow==17.0.0
+certifi==2023.11.17
+nvidia-cufft-cu11==10.9.0.58
+six==1.16.0
+pydantic==2.9.2
+click==8.1.7
+nest-asyncio==1.5.6
+gmpy2==2.1.0
+matplotlib==3.8.2
+scipy==1.11.4
+typing_extensions==4.12.2
+statsmodels==0.14.0
+huggingface-hub==0.25.0
+frozenlist==1.4.1
+gpustat==1.1.1
+nvidia-nvtx-cu11==11.7.91
+safetensors==0.4.5
+stanza==1.9.2
+decorator==5.1.1
+seaborn==0.13.0
+sentencepiece==0.2.0
+PyYAML==6.0.1
+black==24.8.0
+protobuf==4.25.1
+pickleshare==0.7.5
+peft==0.13.0
+triton==2.0.0
+nvidia-cuda-runtime-cu11==11.7.99
+Jinja2==3.1.2
+nvidia-cusolver-cu11==11.4.0.1
+executing==1.2.0
+jupyter_client==8.1.0
+pluggy==1.3.0
+cmake==3.30.3
+pytz==2023.3.post1
+aiohappyeyeballs==2.4.2
+kiwisolver==1.4.5
+py-cpuinfo==9.0.0
+Pillow==10.1.0
+ptyprocess==0.7.0
+importlib_resources==6.4.5
+GitPython==3.1.43
+importlib-metadata==6.0.0
+iniconfig==2.0.0
+scikit-learn==1.3.2
+exceptiongroup==1.1.0
+networkx==2.8.6
+accelerate==1.0.0
+nltk==3.8.1
+shutilwhich==1.1.0
+fonttools==4.45.1
+future==0.18.3
+aiohttp==3.10.6
+wcwidth==0.2.5
+idna==3.6
+filelock==3.12.2
+pathspec==0.12.1
+jupyter_core==5.1.0
+lit==18.1.8
+nvidia-curand-cu11==10.2.10.91
+nvidia-cublas-cu11==11.10.3.66
+nvidia-ml-py==12.560.30
+msgpack==1.1.0
+python-dateutil==2.8.2
+blessed==1.20.0
+packaging==23.0
+gitdb==4.0.11
+yarl==1.13.0
+emoji==2.8.0
+tzdata==2023.3
+cycler==0.12.1
+tornado==6.2
+backcall==0.2.0
+plotnine==0.12.4
+ninja==1.11.1.1
+latex==0.7.0
+wandb==0.18.5
+setproctitle==1.3.3
+threadpoolctl==3.2.0
+requests==2.32.3
+pyparsing==3.1.1
+smmap==5.0.1
+pyzmq==23.0.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+matplotlib-inline==0.1.6
+latexcodec==1.0.0
+ipython==8.0.0
+patsy==0.5.3
+contourpy==1.2.0
+multidict==6.1.0
+mizani==0.9.3
+urllib3==2.1.0
+tokenizers==0.20.0
+MarkupSafe==2.1.2
+pip==24.2
+pexpect==4.8.0
+tqdm==4.66.5
+jedi==0.18.2
+pydantic_core==2.23.4
+tempdir==0.7.1
+mpmath==1.2.1
+setuptools==72.1.0
+pytest==7.4.3
+pure-eval==0.2.2
+psutil==5.9.1
+comm==0.1.2
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+regex==2023.10.3
+platformdirs==2.5.2
+asttokens==2.2.1
+torch==2.0.0
+nvidia-nccl-cu11==2.14.3
+xxhash==3.5.0