Add files using upload-large-folder tool

.gitattributes

@@ -131,3 +131,10 @@ wandb/run-20241105_163244-o1vw2gev/run-o1vw2gev.wandb filter=lfs diff=lfs merge=
 wandb/run-20241115_125218-wtoze6lw/run-wtoze6lw.wandb filter=lfs diff=lfs merge=lfs -text
 wandb/run-20241030_112852-mfvd6tgw/run-mfvd6tgw.wandb filter=lfs diff=lfs merge=lfs -text
 wandb/run-20241031_114700-78zg7gu4/run-78zg7gu4.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241118_234414-wbi0r8f5/run-wbi0r8f5.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241105_163248-ec3mgg2q/run-ec3mgg2q.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241114_090201-6a5c399u/run-6a5c399u.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241031_122113-8ldget07/run-8ldget07.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241129_235241-3vsgtb8n/run-3vsgtb8n.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241106_234111-gz9chn3b/run-gz9chn3b.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241119_135256-3ybits7j/run-3ybits7j.wandb filter=lfs diff=lfs merge=lfs -text

wandb/run-20241030_010305-cstaz19d/files/config.yaml

@@ -0,0 +1,47 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.9.19
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "3":
+                - 13
+                - 23
+                - 55
+            "4": 3.9.19
+            "5": 0.18.5
+            "6": 4.45.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+batch_size:
+    value: 3
+epoch:
+    value: 7
+perturbation:
+    value: reverse_control
+seed:
+    value: 0
+train_set:
+    value: 10M

wandb/run-20241030_010305-cstaz19d/files/output.log

@@ -0,0 +1,4 @@
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 162, in <module>
+    dataset_name = f"babylm_{args.perturbation}_{args.train_zset}_seed{args.seed}"
+AttributeError: 'Namespace' object has no attribute 'train_zset'

wandb/run-20241030_010305-cstaz19d/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-10-30T05:03:05.925719Z",
+  "args":  [
+    "--perturbation",
+    "reverse_control",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "7",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1719200043008"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241030_010305-cstaz19d/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":0}}

wandb/run-20241030_010305-cstaz19d/logs/debug-internal.log

@@ -0,0 +1,16 @@
+{"time":"2024-10-30T01:03:05.927714972-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-10-30T01:03:05.927727572-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_010305-cstaz19d/logs/debug-core.log"}
+{"time":"2024-10-30T01:03:06.033884173-04:00","level":"INFO","msg":"created new stream","id":"cstaz19d"}
+{"time":"2024-10-30T01:03:06.033933893-04:00","level":"INFO","msg":"stream: started","id":"cstaz19d"}
+{"time":"2024-10-30T01:03:06.034003683-04:00","level":"INFO","msg":"sender: started","stream_id":"cstaz19d"}
+{"time":"2024-10-30T01:03:06.033977053-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"cstaz19d"}}
+{"time":"2024-10-30T01:03:06.033959033-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"cstaz19d"}}
+{"time":"2024-10-30T01:03:06.194562967-04:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-10-30T01:03:06.288758935-04:00","level":"INFO","msg":"stream: closing","id":"cstaz19d"}
+{"time":"2024-10-30T01:03:06.288810965-04:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-10-30T01:03:06.289371279-04:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-10-30T01:03:07.676837759-04:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-10-30T01:03:07.809162789-04:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"cstaz19d"}}
+{"time":"2024-10-30T01:03:07.80920396-04:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"cstaz19d"}}
+{"time":"2024-10-30T01:03:07.80922205-04:00","level":"INFO","msg":"sender: closed","stream_id":"cstaz19d"}
+{"time":"2024-10-30T01:03:07.80925996-04:00","level":"INFO","msg":"stream: closed","id":"cstaz19d"}

wandb/run-20241030_010305-cstaz19d/logs/debug.log

@@ -0,0 +1,27 @@
+2024-10-30 01:03:05,924 INFO    MainThread:320656 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-10-30 01:03:05,924 INFO    MainThread:320656 [wandb_setup.py:_flush():79] Configure stats pid to 320656
+2024-10-30 01:03:05,924 INFO    MainThread:320656 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-10-30 01:03:05,924 INFO    MainThread:320656 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-10-30 01:03:05,924 INFO    MainThread:320656 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-30 01:03:05,924 INFO    MainThread:320656 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-30 01:03:05,924 INFO    MainThread:320656 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-10-30 01:03:05,924 INFO    MainThread:320656 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-30 01:03:05,924 INFO    MainThread:320656 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_010305-cstaz19d/logs/debug.log
+2024-10-30 01:03:05,924 INFO    MainThread:320656 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_010305-cstaz19d/logs/debug-internal.log
+2024-10-30 01:03:05,924 INFO    MainThread:320656 [wandb_init.py:init():621] calling init triggers
+2024-10-30 01:03:05,924 INFO    MainThread:320656 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-10-30 01:03:05,924 INFO    MainThread:320656 [wandb_init.py:init():671] starting backend
+2024-10-30 01:03:05,924 INFO    MainThread:320656 [wandb_init.py:init():675] sending inform_init request
+2024-10-30 01:03:05,925 INFO    MainThread:320656 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-30 01:03:05,925 INFO    MainThread:320656 [wandb_init.py:init():688] backend started and connected
+2024-10-30 01:03:05,928 INFO    MainThread:320656 [wandb_init.py:init():783] updated telemetry
+2024-10-30 01:03:05,953 INFO    MainThread:320656 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-10-30 01:03:06,191 INFO    MainThread:320656 [wandb_init.py:init():867] starting run threads in backend
+2024-10-30 01:03:06,286 INFO    MainThread:320656 [wandb_run.py:_console_start():2463] atexit reg
+2024-10-30 01:03:06,286 INFO    MainThread:320656 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-10-30 01:03:06,286 INFO    MainThread:320656 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-10-30 01:03:06,286 INFO    MainThread:320656 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-10-30 01:03:06,287 INFO    MainThread:320656 [wandb_init.py:init():911] run started, returning control to user process
+2024-10-30 01:03:06,287 INFO    MainThread:320656 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_control', 'train_set': '10M', 'batch_size': 3, 'epoch': 7, 'seed': 0}
+2024-10-30 01:03:06,288 WARNING MsgRouterThr:320656 [router.py:message_loop():77] message_loop has been closed

wandb/run-20241030_010641-j5g229e9/files/config.yaml

@@ -0,0 +1,47 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.9.19
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "3":
+                - 13
+                - 23
+                - 55
+            "4": 3.9.19
+            "5": 0.18.5
+            "6": 4.45.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+batch_size:
+    value: 3
+epoch:
+    value: 7
+perturbation:
+    value: reverse_control
+seed:
+    value: 0
+train_set:
+    value: 10M

wandb/run-20241030_010641-j5g229e9/files/output.log

@@ -0,0 +1,4 @@
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 162, in <module>
+    dataset_name = f"babylm_{args.perturbation}_{args.train_zset}_seed{args.seed}"
+AttributeError: 'Namespace' object has no attribute 'train_zset'

wandb/run-20241030_010641-j5g229e9/files/wandb-metadata.json

@@ -0,0 +1,29 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-10-30T05:06:41.323088Z",
+  "args":  [
+    "--perturbation",
+    "reverse_control",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "7",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py"
+}

wandb/run-20241030_010641-j5g229e9/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":1}}

wandb/run-20241030_010641-j5g229e9/logs/debug-internal.log

@@ -0,0 +1,16 @@
+{"time":"2024-10-30T01:06:41.324841553-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-10-30T01:06:41.324850723-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_010641-j5g229e9/logs/debug-core.log"}
+{"time":"2024-10-30T01:06:41.430256903-04:00","level":"INFO","msg":"created new stream","id":"j5g229e9"}
+{"time":"2024-10-30T01:06:41.430292543-04:00","level":"INFO","msg":"stream: started","id":"j5g229e9"}
+{"time":"2024-10-30T01:06:41.430340433-04:00","level":"INFO","msg":"sender: started","stream_id":"j5g229e9"}
+{"time":"2024-10-30T01:06:41.430317333-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"j5g229e9"}}
+{"time":"2024-10-30T01:06:41.430321883-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"j5g229e9"}}
+{"time":"2024-10-30T01:06:43.013728297-04:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-10-30T01:06:43.112897224-04:00","level":"INFO","msg":"stream: closing","id":"j5g229e9"}
+{"time":"2024-10-30T01:06:43.112938124-04:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-10-30T01:06:43.222698323-04:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-10-30T01:06:43.681860685-04:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-10-30T01:06:43.797739476-04:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"j5g229e9"}}
+{"time":"2024-10-30T01:06:43.797790427-04:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"j5g229e9"}}
+{"time":"2024-10-30T01:06:43.797807137-04:00","level":"INFO","msg":"sender: closed","stream_id":"j5g229e9"}
+{"time":"2024-10-30T01:06:43.797873947-04:00","level":"INFO","msg":"stream: closed","id":"j5g229e9"}

wandb/run-20241030_010641-j5g229e9/logs/debug.log

@@ -0,0 +1,27 @@
+2024-10-30 01:06:41,320 INFO    MainThread:321593 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-10-30 01:06:41,320 INFO    MainThread:321593 [wandb_setup.py:_flush():79] Configure stats pid to 321593
+2024-10-30 01:06:41,320 INFO    MainThread:321593 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-10-30 01:06:41,320 INFO    MainThread:321593 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-10-30 01:06:41,320 INFO    MainThread:321593 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-30 01:06:41,321 INFO    MainThread:321593 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-30 01:06:41,321 INFO    MainThread:321593 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-10-30 01:06:41,321 INFO    MainThread:321593 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-30 01:06:41,321 INFO    MainThread:321593 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_010641-j5g229e9/logs/debug.log
+2024-10-30 01:06:41,321 INFO    MainThread:321593 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_010641-j5g229e9/logs/debug-internal.log
+2024-10-30 01:06:41,321 INFO    MainThread:321593 [wandb_init.py:init():621] calling init triggers
+2024-10-30 01:06:41,321 INFO    MainThread:321593 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-10-30 01:06:41,321 INFO    MainThread:321593 [wandb_init.py:init():671] starting backend
+2024-10-30 01:06:41,321 INFO    MainThread:321593 [wandb_init.py:init():675] sending inform_init request
+2024-10-30 01:06:41,322 INFO    MainThread:321593 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-30 01:06:41,322 INFO    MainThread:321593 [wandb_init.py:init():688] backend started and connected
+2024-10-30 01:06:41,325 INFO    MainThread:321593 [wandb_init.py:init():783] updated telemetry
+2024-10-30 01:06:41,349 INFO    MainThread:321593 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-10-30 01:06:43,010 INFO    MainThread:321593 [wandb_init.py:init():867] starting run threads in backend
+2024-10-30 01:06:43,110 INFO    MainThread:321593 [wandb_run.py:_console_start():2463] atexit reg
+2024-10-30 01:06:43,110 INFO    MainThread:321593 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-10-30 01:06:43,110 INFO    MainThread:321593 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-10-30 01:06:43,110 INFO    MainThread:321593 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-10-30 01:06:43,111 INFO    MainThread:321593 [wandb_init.py:init():911] run started, returning control to user process
+2024-10-30 01:06:43,112 INFO    MainThread:321593 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_control', 'train_set': '10M', 'batch_size': 3, 'epoch': 7, 'seed': 0}
+2024-10-30 01:06:43,113 WARNING MsgRouterThr:321593 [router.py:message_loop():77] message_loop has been closed

wandb/run-20241030_011013-230zmi6k/files/config.yaml

@@ -0,0 +1,47 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.9.19
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "3":
+                - 13
+                - 23
+                - 55
+            "4": 3.9.19
+            "5": 0.18.5
+            "6": 4.45.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+batch_size:
+    value: 3
+epoch:
+    value: 7
+perturbation:
+    value: reverse_control
+seed:
+    value: 0
+train_set:
+    value: 10M

wandb/run-20241030_011013-230zmi6k/files/output.log

@@ -0,0 +1,6 @@
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 165, in <module>
+    valid_dataset = dataset['validation']
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/dataset_dict.py", line 72, in __getitem__
+    return super().__getitem__(k)
+KeyError: 'validation'

wandb/run-20241030_011013-230zmi6k/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-10-30T05:10:13.839509Z",
+  "args":  [
+    "--perturbation",
+    "reverse_control",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "7",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1719200362496"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241030_011013-230zmi6k/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":8}}

wandb/run-20241030_011013-230zmi6k/logs/debug-internal.log

@@ -0,0 +1,16 @@
+{"time":"2024-10-30T01:10:13.841784455-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-10-30T01:10:13.841795945-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_011013-230zmi6k/logs/debug-core.log"}
+{"time":"2024-10-30T01:10:13.949562416-04:00","level":"INFO","msg":"created new stream","id":"230zmi6k"}
+{"time":"2024-10-30T01:10:13.949608986-04:00","level":"INFO","msg":"stream: started","id":"230zmi6k"}
+{"time":"2024-10-30T01:10:13.949619536-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"230zmi6k"}}
+{"time":"2024-10-30T01:10:13.949667507-04:00","level":"INFO","msg":"sender: started","stream_id":"230zmi6k"}
+{"time":"2024-10-30T01:10:13.949658147-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"230zmi6k"}}
+{"time":"2024-10-30T01:10:14.410035382-04:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-10-30T01:10:22.003648439-04:00","level":"INFO","msg":"stream: closing","id":"230zmi6k"}
+{"time":"2024-10-30T01:10:22.003672989-04:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-10-30T01:10:22.004086952-04:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-10-30T01:10:22.393017441-04:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-10-30T01:10:22.505006851-04:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"230zmi6k"}}
+{"time":"2024-10-30T01:10:22.505053072-04:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"230zmi6k"}}
+{"time":"2024-10-30T01:10:22.505071632-04:00","level":"INFO","msg":"sender: closed","stream_id":"230zmi6k"}
+{"time":"2024-10-30T01:10:22.505101302-04:00","level":"INFO","msg":"stream: closed","id":"230zmi6k"}

wandb/run-20241030_011013-230zmi6k/logs/debug.log

@@ -0,0 +1,27 @@
+2024-10-30 01:10:13,836 INFO    MainThread:323564 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-10-30 01:10:13,836 INFO    MainThread:323564 [wandb_setup.py:_flush():79] Configure stats pid to 323564
+2024-10-30 01:10:13,836 INFO    MainThread:323564 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-10-30 01:10:13,836 INFO    MainThread:323564 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-10-30 01:10:13,836 INFO    MainThread:323564 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-30 01:10:13,837 INFO    MainThread:323564 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-30 01:10:13,837 INFO    MainThread:323564 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-10-30 01:10:13,837 INFO    MainThread:323564 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-30 01:10:13,837 INFO    MainThread:323564 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_011013-230zmi6k/logs/debug.log
+2024-10-30 01:10:13,837 INFO    MainThread:323564 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_011013-230zmi6k/logs/debug-internal.log
+2024-10-30 01:10:13,837 INFO    MainThread:323564 [wandb_init.py:init():621] calling init triggers
+2024-10-30 01:10:13,837 INFO    MainThread:323564 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-10-30 01:10:13,837 INFO    MainThread:323564 [wandb_init.py:init():671] starting backend
+2024-10-30 01:10:13,837 INFO    MainThread:323564 [wandb_init.py:init():675] sending inform_init request
+2024-10-30 01:10:13,838 INFO    MainThread:323564 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-30 01:10:13,839 INFO    MainThread:323564 [wandb_init.py:init():688] backend started and connected
+2024-10-30 01:10:13,843 INFO    MainThread:323564 [wandb_init.py:init():783] updated telemetry
+2024-10-30 01:10:13,867 INFO    MainThread:323564 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-10-30 01:10:14,407 INFO    MainThread:323564 [wandb_init.py:init():867] starting run threads in backend
+2024-10-30 01:10:14,507 INFO    MainThread:323564 [wandb_run.py:_console_start():2463] atexit reg
+2024-10-30 01:10:14,507 INFO    MainThread:323564 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-10-30 01:10:14,507 INFO    MainThread:323564 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-10-30 01:10:14,507 INFO    MainThread:323564 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-10-30 01:10:14,508 INFO    MainThread:323564 [wandb_init.py:init():911] run started, returning control to user process
+2024-10-30 01:10:14,508 INFO    MainThread:323564 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_control', 'train_set': '10M', 'batch_size': 3, 'epoch': 7, 'seed': 0}
+2024-10-30 01:10:22,003 WARNING MsgRouterThr:323564 [router.py:message_loop():77] message_loop has been closed

wandb/run-20241030_012617-b4q8kp96/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-10-30T05:26:17.478083Z",
+  "args":  [
+    "--perturbation",
+    "reverse_control",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "7",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1709772775424"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241030_013141-xofrmca4/files/config.yaml

@@ -0,0 +1,47 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.9.19
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "3":
+                - 13
+                - 23
+                - 55
+            "4": 3.9.19
+            "5": 0.18.5
+            "6": 4.45.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+batch_size:
+    value: 3
+epoch:
+    value: 7
+perturbation:
+    value: reverse_full
+seed:
+    value: 0
+train_set:
+    value: 10M

wandb/run-20241030_013141-xofrmca4/files/output.log

@@ -0,0 +1,49 @@
+Downloading shards:   0%|                                                                                                                                        | 0/2 [00:00<?, ?it/s]Exception ignored in: <generator object tqdm.__iter__ at 0x7f706464edd0>
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/tqdm/std.py", line 1196, in __iter__
+    self.close()
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/tqdm/std.py", line 1290, in close
+    fp_write('')
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/tqdm/std.py", line 1287, in fp_write
+    self.fp.write(str(s))
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/tqdm/utils.py", line 196, in inner
+    return func(*args, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/wandb/sdk/lib/redirect.py", line 648, in write
+    cb(data)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/wandb/sdk/wandb_run.py", line 2386, in <lambda>
+    lambda data: self._console_raw_callback("stderr", data),
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/wandb/sdk/wandb_run.py", line 403, in wrapper_fn
+    return func(self, *args, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/wandb/sdk/wandb_run.py", line 1547, in _console_raw_callback
+    self._backend.interface.publish_output_raw(name, data)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/wandb/sdk/interface/interface.py", line 721, in publish_output_raw
+    self._publish_output_raw(o)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/wandb/sdk/interface/interface_shared.py", line 78, in _publish_output_raw
+    rec.output_raw.CopyFrom(outdata)
+KeyboardInterrupt:
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 172, in <module>
+    model = AutoModelForCausalLM.from_pretrained(model_name,
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/models/auto/auto_factory.py", line 564, in from_pretrained
+    return model_class.from_pretrained(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/modeling_utils.py", line 3769, in from_pretrained
+    resolved_archive_file, sharded_metadata = get_checkpoint_shard_files(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/utils/hub.py", line 1098, in get_checkpoint_shard_files
+    cached_filename = cached_file(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/utils/hub.py", line 403, in cached_file
+    resolved_file = hf_hub_download(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/utils/_deprecation.py", line 101, in inner_f
+    return f(*args, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn
+    return fn(*args, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1232, in hf_hub_download
+    return _hf_hub_download_to_cache_dir(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1380, in _hf_hub_download_to_cache_dir
+    with WeakFileLock(lock_path):
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/contextlib.py", line 119, in __enter__
+    return next(self.gen)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/utils/_fixes.py", line 98, in WeakFileLock
+    lock.acquire()
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/filelock/_api.py", line 225, in acquire
+    time.sleep(poll_interval)
+KeyboardInterrupt

wandb/run-20241030_013141-xofrmca4/files/requirements.txt

@@ -0,0 +1,147 @@
+funcsigs==1.0.2
+sentry-sdk==2.17.0
+multiprocess==0.70.16
+numpy==1.26.2
+pluralizer==1.2.0
+debugpy==1.6.7
+nvidia-cudnn-cu11==8.5.0.96
+deepspeed==0.15.2
+data==0.4
+pandas==2.1.3
+tomli==2.0.1
+charset-normalizer==3.3.2
+attrs==24.2.0
+aiosignal==1.3.1
+fsspec==2023.10.0
+nvidia-cusparse-cu11==11.7.4.91
+zipp==3.12.0
+mypy-extensions==1.0.0
+datasets==3.0.1
+joblib==1.3.2
+hjson==3.1.0
+traitlets==5.7.1
+stack-data==0.6.0
+transformers==4.45.1
+sympy==1.11.1
+Pygments==2.15.0
+docker-pycreds==0.4.0
+dill==0.3.8
+wheel==0.44.0
+prompt-toolkit==3.0.30
+parso==0.8.3
+ipykernel==6.23.1
+pyarrow==17.0.0
+certifi==2023.11.17
+nvidia-cufft-cu11==10.9.0.58
+six==1.16.0
+pydantic==2.9.2
+click==8.1.7
+nest-asyncio==1.5.6
+gmpy2==2.1.0
+matplotlib==3.8.2
+scipy==1.11.4
+typing_extensions==4.12.2
+statsmodels==0.14.0
+huggingface-hub==0.25.0
+frozenlist==1.4.1
+gpustat==1.1.1
+nvidia-nvtx-cu11==11.7.91
+safetensors==0.4.5
+stanza==1.9.2
+decorator==5.1.1
+seaborn==0.13.0
+sentencepiece==0.2.0
+PyYAML==6.0.1
+black==24.8.0
+protobuf==4.25.1
+pickleshare==0.7.5
+peft==0.13.0
+triton==2.0.0
+nvidia-cuda-runtime-cu11==11.7.99
+Jinja2==3.1.2
+nvidia-cusolver-cu11==11.4.0.1
+executing==1.2.0
+jupyter_client==8.1.0
+pluggy==1.3.0
+cmake==3.30.3
+pytz==2023.3.post1
+aiohappyeyeballs==2.4.2
+kiwisolver==1.4.5
+py-cpuinfo==9.0.0
+Pillow==10.1.0
+ptyprocess==0.7.0
+importlib_resources==6.4.5
+GitPython==3.1.43
+importlib-metadata==6.0.0
+iniconfig==2.0.0
+scikit-learn==1.3.2
+exceptiongroup==1.1.0
+networkx==2.8.6
+accelerate==1.0.0
+nltk==3.8.1
+shutilwhich==1.1.0
+fonttools==4.45.1
+future==0.18.3
+aiohttp==3.10.6
+wcwidth==0.2.5
+idna==3.6
+filelock==3.12.2
+pathspec==0.12.1
+jupyter_core==5.1.0
+lit==18.1.8
+nvidia-curand-cu11==10.2.10.91
+nvidia-cublas-cu11==11.10.3.66
+nvidia-ml-py==12.560.30
+msgpack==1.1.0
+python-dateutil==2.8.2
+blessed==1.20.0
+packaging==23.0
+gitdb==4.0.11
+yarl==1.13.0
+emoji==2.8.0
+tzdata==2023.3
+cycler==0.12.1
+tornado==6.2
+backcall==0.2.0
+plotnine==0.12.4
+ninja==1.11.1.1
+latex==0.7.0
+wandb==0.18.5
+setproctitle==1.3.3
+threadpoolctl==3.2.0
+requests==2.32.3
+pyparsing==3.1.1
+smmap==5.0.1
+pyzmq==23.0.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+matplotlib-inline==0.1.6
+latexcodec==1.0.0
+ipython==8.0.0
+patsy==0.5.3
+contourpy==1.2.0
+multidict==6.1.0
+mizani==0.9.3
+urllib3==2.1.0
+tokenizers==0.20.0
+MarkupSafe==2.1.2
+pip==24.2
+pexpect==4.8.0
+tqdm==4.66.5
+jedi==0.18.2
+pydantic_core==2.23.4
+tempdir==0.7.1
+mpmath==1.2.1
+setuptools==72.1.0
+pytest==7.4.3
+pure-eval==0.2.2
+psutil==5.9.1
+comm==0.1.2
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+regex==2023.10.3
+platformdirs==2.5.2
+asttokens==2.2.1
+torch==2.0.0
+nvidia-nccl-cu11==2.14.3
+xxhash==3.5.0

wandb/run-20241030_013141-xofrmca4/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-10-30T05:31:41.892494Z",
+  "args":  [
+    "--perturbation",
+    "reverse_full",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "7",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1709824425984"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241030_013141-xofrmca4/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":94}}

wandb/run-20241030_013141-xofrmca4/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2024-10-30T01:31:41.89497181-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-10-30T01:31:41.89498696-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_013141-xofrmca4/logs/debug-core.log"}
+{"time":"2024-10-30T01:31:42.003012506-04:00","level":"INFO","msg":"created new stream","id":"xofrmca4"}
+{"time":"2024-10-30T01:31:42.003051536-04:00","level":"INFO","msg":"stream: started","id":"xofrmca4"}
+{"time":"2024-10-30T01:31:42.003070586-04:00","level":"INFO","msg":"sender: started","stream_id":"xofrmca4"}
+{"time":"2024-10-30T01:31:42.003072466-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"xofrmca4"}}
+{"time":"2024-10-30T01:31:42.003066286-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"xofrmca4"}}
+{"time":"2024-10-30T01:31:42.198776353-04:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-10-30T01:33:16.60059966-04:00","level":"INFO","msg":"stream: closing","id":"xofrmca4"}
+{"time":"2024-10-30T01:33:16.60066801-04:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-10-30T01:33:16.601290814-04:00","level":"INFO","msg":"Stopped system monitor"}

wandb/run-20241030_013141-xofrmca4/logs/debug.log

@@ -0,0 +1,27 @@
+2024-10-30 01:31:41,890 INFO    MainThread:335754 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-10-30 01:31:41,890 INFO    MainThread:335754 [wandb_setup.py:_flush():79] Configure stats pid to 335754
+2024-10-30 01:31:41,890 INFO    MainThread:335754 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-10-30 01:31:41,890 INFO    MainThread:335754 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-10-30 01:31:41,890 INFO    MainThread:335754 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-30 01:31:41,890 INFO    MainThread:335754 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-30 01:31:41,890 INFO    MainThread:335754 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-10-30 01:31:41,890 INFO    MainThread:335754 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-30 01:31:41,890 INFO    MainThread:335754 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_013141-xofrmca4/logs/debug.log
+2024-10-30 01:31:41,891 INFO    MainThread:335754 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_013141-xofrmca4/logs/debug-internal.log
+2024-10-30 01:31:41,891 INFO    MainThread:335754 [wandb_init.py:init():621] calling init triggers
+2024-10-30 01:31:41,891 INFO    MainThread:335754 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-10-30 01:31:41,891 INFO    MainThread:335754 [wandb_init.py:init():671] starting backend
+2024-10-30 01:31:41,891 INFO    MainThread:335754 [wandb_init.py:init():675] sending inform_init request
+2024-10-30 01:31:41,891 INFO    MainThread:335754 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-30 01:31:41,892 INFO    MainThread:335754 [wandb_init.py:init():688] backend started and connected
+2024-10-30 01:31:41,895 INFO    MainThread:335754 [wandb_init.py:init():783] updated telemetry
+2024-10-30 01:31:41,926 INFO    MainThread:335754 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-10-30 01:31:42,195 INFO    MainThread:335754 [wandb_init.py:init():867] starting run threads in backend
+2024-10-30 01:31:42,300 INFO    MainThread:335754 [wandb_run.py:_console_start():2463] atexit reg
+2024-10-30 01:31:42,301 INFO    MainThread:335754 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-10-30 01:31:42,301 INFO    MainThread:335754 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-10-30 01:31:42,301 INFO    MainThread:335754 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-10-30 01:31:42,302 INFO    MainThread:335754 [wandb_init.py:init():911] run started, returning control to user process
+2024-10-30 01:31:42,302 INFO    MainThread:335754 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_full', 'train_set': '10M', 'batch_size': 3, 'epoch': 7, 'seed': 0}
+2024-10-30 01:33:16,600 WARNING MsgRouterThr:335754 [router.py:message_loop():77] message_loop has been closed

wandb/run-20241030_112852-muula7m7/files/config.yaml

@@ -0,0 +1,529 @@
+_name_or_path:
+    value: meta-llama/Llama-3.2-3B
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m:
+            - "1": train/global_step
+              "6":
+                - 3
+              "7": []
+            - "1": train/loss
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": train/learning_rate
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": eval/samples_per_second
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": eval/steps_per_second
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": train/epoch
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": train/grad_norm
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": eval/runtime
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": eval/loss
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+        python_version: 3.9.19
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "3":
+                - 2
+                - 7
+                - 13
+                - 19
+                - 23
+                - 55
+                - 62
+                - 66
+            "4": 3.9.19
+            "5": 0.18.5
+            "6": 4.45.1
+            "8":
+                - 5
+            "9":
+                "1": transformers_trainer
+            "12": 0.18.5
+            "13": linux-x86_64
+accelerator_config:
+    value:
+        dispatch_batches: null
+        even_batches: true
+        gradient_accumulation_kwargs: null
+        non_blocking: false
+        split_batches: false
+        use_seedable_sampler: true
+adafactor:
+    value: false
+adam_beta1:
+    value: 0.9
+adam_beta2:
+    value: 0.999
+adam_epsilon:
+    value: 1e-08
+add_cross_attention:
+    value: false
+architectures:
+    value:
+        - LlamaForCausalLM
+attention_bias:
+    value: false
+attention_dropout:
+    value: 0
+auto_find_batch_size:
+    value: false
+bad_words_ids:
+    value: null
+batch_eval_metrics:
+    value: false
+batch_size:
+    value: 3
+begin_suppress_tokens:
+    value: null
+bf16:
+    value: false
+bf16_full_eval:
+    value: false
+bos_token_id:
+    value: 128000
+chunk_size_feed_forward:
+    value: 0
+cross_attention_hidden_size:
+    value: null
+data_seed:
+    value: null
+dataloader_drop_last:
+    value: false
+dataloader_num_workers:
+    value: 0
+dataloader_persistent_workers:
+    value: false
+dataloader_pin_memory:
+    value: true
+dataloader_prefetch_factor:
+    value: null
+ddp_backend:
+    value: null
+ddp_broadcast_buffers:
+    value: null
+ddp_bucket_cap_mb:
+    value: null
+ddp_find_unused_parameters:
+    value: null
+ddp_timeout:
+    value: 1800
+debug:
+    value: []
+decoder_start_token_id:
+    value: null
+deepspeed:
+    value: deepspeed_config/train_dp_config.json
+disable_tqdm:
+    value: false
+dispatch_batches:
+    value: null
+diversity_penalty:
+    value: 0
+do_eval:
+    value: true
+do_predict:
+    value: false
+do_sample:
+    value: false
+do_train:
+    value: false
+early_stopping:
+    value: false
+encoder_no_repeat_ngram_size:
+    value: 0
+eos_token_id:
+    value: 128001
+epoch:
+    value: 3
+eval_accumulation_steps:
+    value: null
+eval_delay:
+    value: 0
+eval_do_concat_batches:
+    value: true
+eval_on_start:
+    value: false
+eval_steps:
+    value: 10
+eval_strategy:
+    value: steps
+eval_use_gather_object:
+    value: false
+evaluation_strategy:
+    value: steps
+exponential_decay_length_penalty:
+    value: null
+finetuning_task:
+    value: null
+forced_bos_token_id:
+    value: null
+forced_eos_token_id:
+    value: null
+fp16:
+    value: true
+fp16_backend:
+    value: auto
+fp16_full_eval:
+    value: false
+fp16_opt_level:
+    value: O1
+fsdp:
+    value: []
+fsdp_config:
+    value:
+        min_num_params: 0
+        xla: false
+        xla_fsdp_grad_ckpt: false
+        xla_fsdp_v2: false
+fsdp_min_num_params:
+    value: 0
+fsdp_transformer_layer_cls_to_wrap:
+    value: null
+full_determinism:
+    value: false
+gradient_accumulation_steps:
+    value: 1
+gradient_checkpointing:
+    value: false
+gradient_checkpointing_kwargs:
+    value: null
+greater_is_better:
+    value: null
+group_by_length:
+    value: false
+half_precision_backend:
+    value: auto
+head_dim:
+    value: 128
+hidden_act:
+    value: silu
+hidden_size:
+    value: 3072
+hub_always_push:
+    value: false
+hub_model_id:
+    value: null
+hub_private_repo:
+    value: false
+hub_strategy:
+    value: every_save
+hub_token:
+    value: <HUB_TOKEN>
+id2label:
+    value:
+        "0": LABEL_0
+        "1": LABEL_1
+ignore_data_skip:
+    value: false
+include_inputs_for_metrics:
+    value: false
+include_num_input_tokens_seen:
+    value: false
+include_tokens_per_second:
+    value: false
+initializer_range:
+    value: 0.02
+intermediate_size:
+    value: 8192
+is_decoder:
+    value: false
+is_encoder_decoder:
+    value: false
+jit_mode_eval:
+    value: false
+label_names:
+    value: null
+label_smoothing_factor:
+    value: 0
+label2id:
+    value:
+        LABEL_0: 0
+        LABEL_1: 1
+learning_rate:
+    value: 5e-05
+length_column_name:
+    value: length
+length_penalty:
+    value: 1
+load_best_model_at_end:
+    value: false
+local_rank:
+    value: 0
+log_level:
+    value: passive
+log_level_replica:
+    value: warning
+log_on_each_node:
+    value: true
+logging_dir:
+    value: ./logs
+logging_first_step:
+    value: false
+logging_nan_inf_filter:
+    value: true
+logging_steps:
+    value: 1
+logging_strategy:
+    value: steps
+lr_scheduler_type:
+    value: linear
+max_grad_norm:
+    value: 1
+max_length:
+    value: 20
+max_position_embeddings:
+    value: 131072
+max_steps:
+    value: -1
+metric_for_best_model:
+    value: null
+min_length:
+    value: 0
+mlp_bias:
+    value: false
+model/num_parameters:
+    value: 3212749824
+model_type:
+    value: llama
+mp_parameters:
+    value: ""
+neftune_noise_alpha:
+    value: null
+no_cuda:
+    value: false
+no_repeat_ngram_size:
+    value: 0
+num_attention_heads:
+    value: 24
+num_beam_groups:
+    value: 1
+num_beams:
+    value: 1
+num_hidden_layers:
+    value: 28
+num_key_value_heads:
+    value: 8
+num_return_sequences:
+    value: 1
+num_train_epochs:
+    value: 3
+optim:
+    value: adamw_torch
+optim_args:
+    value: null
+optim_target_modules:
+    value: null
+output_attentions:
+    value: false
+output_dir:
+    value: ./checkpoints/Llama-3.2-3B/babylm_reverse_control_10M_seed0/runs
+output_hidden_states:
+    value: false
+output_scores:
+    value: false
+overwrite_output_dir:
+    value: false
+pad_token_id:
+    value: null
+past_index:
+    value: -1
+per_device_eval_batch_size:
+    value: 8
+per_device_train_batch_size:
+    value: 3
+per_gpu_eval_batch_size:
+    value: null
+per_gpu_train_batch_size:
+    value: null
+perturbation:
+    value: reverse_control
+prediction_loss_only:
+    value: false
+prefix:
+    value: null
+pretraining_tp:
+    value: 1
+problem_type:
+    value: null
+push_to_hub:
+    value: false
+push_to_hub_model_id:
+    value: null
+push_to_hub_organization:
+    value: null
+push_to_hub_token:
+    value: <PUSH_TO_HUB_TOKEN>
+ray_scope:
+    value: last
+remove_invalid_values:
+    value: false
+remove_unused_columns:
+    value: true
+repetition_penalty:
+    value: 1
+report_to:
+    value:
+        - wandb
+restore_callback_states_from_checkpoint:
+    value: false
+resume_from_checkpoint:
+    value: null
+return_dict:
+    value: true
+return_dict_in_generate:
+    value: false
+rms_norm_eps:
+    value: 1e-05
+rope_scaling:
+    value:
+        factor: 32
+        high_freq_factor: 4
+        low_freq_factor: 1
+        original_max_position_embeddings: 8192
+        rope_type: llama3
+rope_theta:
+    value: 500000
+run_name:
+    value: ./checkpoints/Llama-3.2-3B/babylm_reverse_control_10M_seed0/runs
+save_on_each_node:
+    value: false
+save_only_model:
+    value: false
+save_safetensors:
+    value: true
+save_steps:
+    value: 150
+save_strategy:
+    value: steps
+save_total_limit:
+    value: null
+seed:
+    value: 0
+sep_token_id:
+    value: null
+skip_memory_metrics:
+    value: true
+split_batches:
+    value: null
+suppress_tokens:
+    value: null
+task_specific_params:
+    value: null
+temperature:
+    value: 1
+tf_legacy_loss:
+    value: false
+tf32:
+    value: null
+tie_encoder_decoder:
+    value: false
+tie_word_embeddings:
+    value: true
+tokenizer_class:
+    value: null
+top_k:
+    value: 50
+top_p:
+    value: 1
+torch_compile:
+    value: false
+torch_compile_backend:
+    value: null
+torch_compile_mode:
+    value: null
+torch_dtype:
+    value: bfloat16
+torch_empty_cache_steps:
+    value: null
+torchdynamo:
+    value: null
+torchscript:
+    value: false
+tpu_metrics_debug:
+    value: false
+tpu_num_cores:
+    value: null
+train_set:
+    value: 10M
+transformers_version:
+    value: 4.45.1
+typical_p:
+    value: 1
+use_bfloat16:
+    value: false
+use_cache:
+    value: true
+use_cpu:
+    value: false
+use_ipex:
+    value: false
+use_legacy_prediction_loop:
+    value: false
+use_liger_kernel:
+    value: false
+use_mps_device:
+    value: false
+vocab_size:
+    value: 128256
+warmup_ratio:
+    value: 0
+warmup_steps:
+    value: 0
+weight_decay:
+    value: 0

wandb/run-20241030_112852-muula7m7/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-10-30T15:28:52.931579Z",
+  "args":  [
+    "--perturbation",
+    "reverse_control",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "3",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1710831611904"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241030_112852-muula7m7/files/wandb-summary.json

@@ -0,0 +1 @@
+{"train/loss":0.9444,"train/global_step":2922,"eval/samples_per_second":52.113,"train/grad_norm":1.6805195808410645,"_wandb":{"runtime":23503},"train_steps_per_second":0.126,"eval/runtime":11.5134,"train_loss":1.3221683262106003,"_step":3214,"_runtime":23503.02541121,"total_flos":9.10877809415553e+17,"train/epoch":3,"eval/loss":1.954732060432434,"train_samples_per_second":2.258,"eval/steps_per_second":1.129,"train_runtime":23278.6909,"train/learning_rate":3.2511978097193705e-07,"_timestamp":1.7303256359141476e+09}

wandb/run-20241030_112852-muula7m7/logs/debug-internal.log

@@ -0,0 +1,17 @@
+{"time":"2024-10-30T11:28:52.934023309-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-10-30T11:28:52.934032969-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_112852-muula7m7/logs/debug-core.log"}
+{"time":"2024-10-30T11:28:53.039288912-04:00","level":"INFO","msg":"created new stream","id":"muula7m7"}
+{"time":"2024-10-30T11:28:53.039312182-04:00","level":"INFO","msg":"stream: started","id":"muula7m7"}
+{"time":"2024-10-30T11:28:53.039350533-04:00","level":"INFO","msg":"sender: started","stream_id":"muula7m7"}
+{"time":"2024-10-30T11:28:53.039342942-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"muula7m7"}}
+{"time":"2024-10-30T11:28:53.039369793-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"muula7m7"}}
+{"time":"2024-10-30T11:28:53.205972504-04:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-10-30T18:00:35.957092491-04:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-10-30T18:00:35.979794251-04:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-10-30T18:00:36.956163819-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"saving job artifact","runtime_seconds":0.333898538}],"total_operations":1}}
+{"time":"2024-10-30T18:00:38.038580508-04:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-10-30T18:00:39.246501757-04:00","level":"INFO","msg":"stream: closing","id":"muula7m7"}
+{"time":"2024-10-30T18:00:39.246544028-04:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"muula7m7"}}
+{"time":"2024-10-30T18:00:39.246572358-04:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"muula7m7"}}
+{"time":"2024-10-30T18:00:39.246591208-04:00","level":"INFO","msg":"sender: closed","stream_id":"muula7m7"}
+{"time":"2024-10-30T18:00:39.246666498-04:00","level":"INFO","msg":"stream: closed","id":"muula7m7"}

wandb/run-20241030_112852-muula7m7/logs/debug.log

@@ -0,0 +1,36 @@
+2024-10-30 11:28:52,929 INFO    MainThread:367764 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-10-30 11:28:52,929 INFO    MainThread:367764 [wandb_setup.py:_flush():79] Configure stats pid to 367764
+2024-10-30 11:28:52,930 INFO    MainThread:367764 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-10-30 11:28:52,930 INFO    MainThread:367764 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-10-30 11:28:52,930 INFO    MainThread:367764 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-30 11:28:52,930 INFO    MainThread:367764 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-30 11:28:52,930 INFO    MainThread:367764 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-10-30 11:28:52,930 INFO    MainThread:367764 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-30 11:28:52,930 INFO    MainThread:367764 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_112852-muula7m7/logs/debug.log
+2024-10-30 11:28:52,930 INFO    MainThread:367764 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_112852-muula7m7/logs/debug-internal.log
+2024-10-30 11:28:52,930 INFO    MainThread:367764 [wandb_init.py:init():621] calling init triggers
+2024-10-30 11:28:52,930 INFO    MainThread:367764 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-10-30 11:28:52,930 INFO    MainThread:367764 [wandb_init.py:init():671] starting backend
+2024-10-30 11:28:52,930 INFO    MainThread:367764 [wandb_init.py:init():675] sending inform_init request
+2024-10-30 11:28:52,931 INFO    MainThread:367764 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-30 11:28:52,931 INFO    MainThread:367764 [wandb_init.py:init():688] backend started and connected
+2024-10-30 11:28:52,934 INFO    MainThread:367764 [wandb_init.py:init():783] updated telemetry
+2024-10-30 11:28:52,972 INFO    MainThread:367764 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-10-30 11:28:53,202 INFO    MainThread:367764 [wandb_init.py:init():867] starting run threads in backend
+2024-10-30 11:28:53,317 INFO    MainThread:367764 [wandb_run.py:_console_start():2463] atexit reg
+2024-10-30 11:28:53,317 INFO    MainThread:367764 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-10-30 11:28:53,317 INFO    MainThread:367764 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-10-30 11:28:53,317 INFO    MainThread:367764 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-10-30 11:28:53,319 INFO    MainThread:367764 [wandb_init.py:init():911] run started, returning control to user process
+2024-10-30 11:28:53,319 INFO    MainThread:367764 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_control', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0}
+2024-10-30 11:32:37,225 INFO    MainThread:367764 [wandb_run.py:_config_callback():1390] config_cb None None {'vocab_size': 128256, 'max_position_embeddings': 131072, 'hidden_size': 3072, 'intermediate_size': 8192, 'num_hidden_layers': 28, 'num_attention_heads': 24, 'num_key_value_heads': 8, 'hidden_act': 'silu', 'initializer_range': 0.02, 'rms_norm_eps': 1e-05, 'pretraining_tp': 1, 'use_cache': True, 'rope_theta': 500000.0, 'rope_scaling': {'factor': 32.0, 'high_freq_factor': 4.0, 'low_freq_factor': 1.0, 'original_max_position_embeddings': 8192, 'rope_type': 'llama3'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 128, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'bfloat16', 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': True, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['LlamaForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 128000, 'pad_token_id': None, 'eos_token_id': 128001, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'meta-llama/Llama-3.2-3B', 'transformers_version': '4.45.1', 'model_type': 'llama', 'output_dir': './checkpoints/Llama-3.2-3B/babylm_reverse_control_10M_seed0/runs', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': True, 'do_predict': False, 'eval_strategy': 'steps', 'prediction_loss_only': False, 'per_device_train_batch_size': 3, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'eval_delay': 0, 'torch_empty_cache_steps': None, 'learning_rate': 5e-05, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 3, 'max_steps': -1, 'lr_scheduler_type': 'linear', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.0, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': './logs', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 1, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 150, 'save_total_limit': None, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 0, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': False, 'fp16': True, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 10, 'dataloader_num_workers': 0, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': './checkpoints/Llama-3.2-3B/babylm_reverse_control_10M_seed0/runs', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': 'deepspeed_config/train_dp_config.json', 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'evaluation_strategy': 'steps', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False, 'use_liger_kernel': False, 'eval_use_gather_object': False}
+2024-10-30 11:32:37,228 INFO    MainThread:367764 [wandb_config.py:__setitem__():154] config set model/num_parameters = 3212749824 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x7fa2ae554e80>>
+2024-10-30 11:32:37,228 INFO    MainThread:367764 [wandb_run.py:_config_callback():1390] config_cb model/num_parameters 3212749824 None
+2024-10-30 18:00:35,945 INFO    MainThread:367764 [wandb_run.py:_finish():2158] finishing run yaning1001-dartmouth-college/impossible_llm_reverse/muula7m7
+2024-10-30 18:00:35,954 INFO    MainThread:367764 [wandb_run.py:_atexit_cleanup():2426] got exitcode: 0
+2024-10-30 18:00:35,954 INFO    MainThread:367764 [wandb_run.py:_restore():2408] restore
+2024-10-30 18:00:35,955 INFO    MainThread:367764 [wandb_run.py:_restore():2414] restore done
+2024-10-30 18:00:39,173 INFO    MainThread:367764 [wandb_run.py:_footer_history_summary_info():3975] rendering history
+2024-10-30 18:00:39,175 INFO    MainThread:367764 [wandb_run.py:_footer_history_summary_info():4007] rendering summary
+2024-10-30 18:00:39,245 INFO    MainThread:367764 [wandb_run.py:_footer_sync_info():3934] logging synced files

wandb/run-20241030_233740-m178kn64/files/output.log

@@ -0,0 +1,13 @@
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:05<00:00,  2.66s/it]
+tokenized_valid: Dataset({
+    features: ['input_ids', 'attention_mask'],
+    num_rows: 600
+})
+/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
+  warnings.warn(
+[2024-10-30 23:37:51,405] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2024-10-30 23:37:59,364] [INFO] [comm.py:652:init_distributed] cdb=None
+Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination
+Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...
+Loading extension module cpu_adam...
+Time to load cpu_adam op: 4.663320779800415 seconds

wandb/run-20241030_233740-m178kn64/files/requirements.txt

@@ -0,0 +1,147 @@
+funcsigs==1.0.2
+sentry-sdk==2.17.0
+multiprocess==0.70.16
+numpy==1.26.2
+pluralizer==1.2.0
+debugpy==1.6.7
+nvidia-cudnn-cu11==8.5.0.96
+deepspeed==0.15.2
+data==0.4
+pandas==2.1.3
+tomli==2.0.1
+charset-normalizer==3.3.2
+attrs==24.2.0
+aiosignal==1.3.1
+fsspec==2023.10.0
+nvidia-cusparse-cu11==11.7.4.91
+zipp==3.12.0
+mypy-extensions==1.0.0
+datasets==3.0.1
+joblib==1.3.2
+hjson==3.1.0
+traitlets==5.7.1
+stack-data==0.6.0
+transformers==4.45.1
+sympy==1.11.1
+Pygments==2.15.0
+docker-pycreds==0.4.0
+dill==0.3.8
+wheel==0.44.0
+prompt-toolkit==3.0.30
+parso==0.8.3
+ipykernel==6.23.1
+pyarrow==17.0.0
+certifi==2023.11.17
+nvidia-cufft-cu11==10.9.0.58
+six==1.16.0
+pydantic==2.9.2
+click==8.1.7
+nest-asyncio==1.5.6
+gmpy2==2.1.0
+matplotlib==3.8.2
+scipy==1.11.4
+typing_extensions==4.12.2
+statsmodels==0.14.0
+huggingface-hub==0.25.0
+frozenlist==1.4.1
+gpustat==1.1.1
+nvidia-nvtx-cu11==11.7.91
+safetensors==0.4.5
+stanza==1.9.2
+decorator==5.1.1
+seaborn==0.13.0
+sentencepiece==0.2.0
+PyYAML==6.0.1
+black==24.8.0
+protobuf==4.25.1
+pickleshare==0.7.5
+peft==0.13.0
+triton==2.0.0
+nvidia-cuda-runtime-cu11==11.7.99
+Jinja2==3.1.2
+nvidia-cusolver-cu11==11.4.0.1
+executing==1.2.0
+jupyter_client==8.1.0
+pluggy==1.3.0
+cmake==3.30.3
+pytz==2023.3.post1
+aiohappyeyeballs==2.4.2
+kiwisolver==1.4.5
+py-cpuinfo==9.0.0
+Pillow==10.1.0
+ptyprocess==0.7.0
+importlib_resources==6.4.5
+GitPython==3.1.43
+importlib-metadata==6.0.0
+iniconfig==2.0.0
+scikit-learn==1.3.2
+exceptiongroup==1.1.0
+networkx==2.8.6
+accelerate==1.0.0
+nltk==3.8.1
+shutilwhich==1.1.0
+fonttools==4.45.1
+future==0.18.3
+aiohttp==3.10.6
+wcwidth==0.2.5
+idna==3.6
+filelock==3.12.2
+pathspec==0.12.1
+jupyter_core==5.1.0
+lit==18.1.8
+nvidia-curand-cu11==10.2.10.91
+nvidia-cublas-cu11==11.10.3.66
+nvidia-ml-py==12.560.30
+msgpack==1.1.0
+python-dateutil==2.8.2
+blessed==1.20.0
+packaging==23.0
+gitdb==4.0.11
+yarl==1.13.0
+emoji==2.8.0
+tzdata==2023.3
+cycler==0.12.1
+tornado==6.2
+backcall==0.2.0
+plotnine==0.12.4
+ninja==1.11.1.1
+latex==0.7.0
+wandb==0.18.5
+setproctitle==1.3.3
+threadpoolctl==3.2.0
+requests==2.32.3
+pyparsing==3.1.1
+smmap==5.0.1
+pyzmq==23.0.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+matplotlib-inline==0.1.6
+latexcodec==1.0.0
+ipython==8.0.0
+patsy==0.5.3
+contourpy==1.2.0
+multidict==6.1.0
+mizani==0.9.3
+urllib3==2.1.0
+tokenizers==0.20.0
+MarkupSafe==2.1.2
+pip==24.2
+pexpect==4.8.0
+tqdm==4.66.5
+jedi==0.18.2
+pydantic_core==2.23.4
+tempdir==0.7.1
+mpmath==1.2.1
+setuptools==72.1.0
+pytest==7.4.3
+pure-eval==0.2.2
+psutil==5.9.1
+comm==0.1.2
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+regex==2023.10.3
+platformdirs==2.5.2
+asttokens==2.2.1
+torch==2.0.0
+nvidia-nccl-cu11==2.14.3
+xxhash==3.5.0

wandb/run-20241030_233740-m178kn64/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-10-31T03:37:40.852588Z",
+  "args":  [
+    "--perturbation",
+    "reverse_control",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "3",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1711065919488"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241030_233740-m178kn64/logs/debug-internal.log

@@ -0,0 +1,8 @@
+{"time":"2024-10-30T23:37:40.854548305-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-10-30T23:37:40.854558095-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_233740-m178kn64/logs/debug-core.log"}
+{"time":"2024-10-30T23:37:40.966779526-04:00","level":"INFO","msg":"created new stream","id":"m178kn64"}
+{"time":"2024-10-30T23:37:40.966817216-04:00","level":"INFO","msg":"stream: started","id":"m178kn64"}
+{"time":"2024-10-30T23:37:40.966891757-04:00","level":"INFO","msg":"sender: started","stream_id":"m178kn64"}
+{"time":"2024-10-30T23:37:40.966938127-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"m178kn64"}}
+{"time":"2024-10-30T23:37:40.966903677-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"m178kn64"}}
+{"time":"2024-10-30T23:37:41.40409799-04:00","level":"INFO","msg":"Starting system monitor"}

wandb/run-20241030_233740-m178kn64/logs/debug.log

@@ -0,0 +1,26 @@
+2024-10-30 23:37:40,849 INFO    MainThread:464536 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-10-30 23:37:40,849 INFO    MainThread:464536 [wandb_setup.py:_flush():79] Configure stats pid to 464536
+2024-10-30 23:37:40,849 INFO    MainThread:464536 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-10-30 23:37:40,849 INFO    MainThread:464536 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-10-30 23:37:40,849 INFO    MainThread:464536 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-30 23:37:40,849 INFO    MainThread:464536 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-30 23:37:40,849 INFO    MainThread:464536 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-10-30 23:37:40,849 INFO    MainThread:464536 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-30 23:37:40,849 INFO    MainThread:464536 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_233740-m178kn64/logs/debug.log
+2024-10-30 23:37:40,849 INFO    MainThread:464536 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_233740-m178kn64/logs/debug-internal.log
+2024-10-30 23:37:40,850 INFO    MainThread:464536 [wandb_init.py:init():621] calling init triggers
+2024-10-30 23:37:40,850 INFO    MainThread:464536 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-10-30 23:37:40,850 INFO    MainThread:464536 [wandb_init.py:init():671] starting backend
+2024-10-30 23:37:40,850 INFO    MainThread:464536 [wandb_init.py:init():675] sending inform_init request
+2024-10-30 23:37:40,851 INFO    MainThread:464536 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-30 23:37:40,852 INFO    MainThread:464536 [wandb_init.py:init():688] backend started and connected
+2024-10-30 23:37:40,856 INFO    MainThread:464536 [wandb_init.py:init():783] updated telemetry
+2024-10-30 23:37:40,886 INFO    MainThread:464536 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-10-30 23:37:41,401 INFO    MainThread:464536 [wandb_init.py:init():867] starting run threads in backend
+2024-10-30 23:37:42,835 INFO    MainThread:464536 [wandb_run.py:_console_start():2463] atexit reg
+2024-10-30 23:37:42,835 INFO    MainThread:464536 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-10-30 23:37:42,836 INFO    MainThread:464536 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-10-30 23:37:42,836 INFO    MainThread:464536 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-10-30 23:37:42,860 INFO    MainThread:464536 [wandb_init.py:init():911] run started, returning control to user process
+2024-10-30 23:37:42,860 INFO    MainThread:464536 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_control', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0}

wandb/run-20241031_122005-hpmt1tka/files/config.yaml

@@ -0,0 +1,49 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.9.19
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "3":
+                - 13
+                - 23
+                - 55
+            "4": 3.9.19
+            "5": 0.18.5
+            "6": 4.45.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+batch_size:
+    value: 3
+epoch:
+    value: 6
+lr:
+    value: 5e-06
+perturbation:
+    value: reverse_full
+seed:
+    value: 0
+train_set:
+    value: 10M

wandb/run-20241031_122005-hpmt1tka/files/output.log

@@ -0,0 +1,75 @@
+config.json: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 844/844 [00:00<00:00, 235kB/s]
+Downloading shards:   0%|                                                                                                                                        | 0/2 [00:22<?, ?it/s]
+Traceback (most recent call last): 19%|████████████████████▉                                                                                       | 965M/4.97G [00:22<01:33, 42.6MB/s]
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 541, in http_get
+    for chunk in r.iter_content(chunk_size=constants.DOWNLOAD_CHUNK_SIZE):
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/requests/models.py", line 820, in generate
+    yield from self.raw.stream(chunk_size, decode_content=True)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/urllib3/response.py", line 934, in stream
+    data = self.read(amt=amt, decode_content=decode_content)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/urllib3/response.py", line 877, in read
+    data = self._raw_read(amt)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/urllib3/response.py", line 812, in _raw_read
+    data = self._fp_read(amt) if not fp_closed else b""
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/urllib3/response.py", line 789, in _fp_read
+    data = self._fp.read(chunk_amt)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/http/client.py", line 463, in read
+    n = self.readinto(b)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/http/client.py", line 507, in readinto
+    n = self.fp.readinto(b)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/socket.py", line 704, in readinto
+    return self._sock.recv_into(b)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/ssl.py", line 1275, in recv_into
+    return self.read(nbytes, buffer)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/ssl.py", line 1133, in read
+    return self._sslobj.read(len, buffer)
+KeyboardInterrupt
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 173, in <module>
+    model = AutoModelForCausalLM.from_pretrained(model_name,
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/models/auto/auto_factory.py", line 564, in from_pretrained
+    return model_class.from_pretrained(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/modeling_utils.py", line 3769, in from_pretrained
+    resolved_archive_file, sharded_metadata = get_checkpoint_shard_files(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/utils/hub.py", line 1098, in get_checkpoint_shard_files
+    cached_filename = cached_file(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/utils/hub.py", line 403, in cached_file
+    resolved_file = hf_hub_download(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/utils/_deprecation.py", line 101, in inner_f
+    return f(*args, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn
+    return fn(*args, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1232, in hf_hub_download
+    return _hf_hub_download_to_cache_dir(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1381, in _hf_hub_download_to_cache_dir
+    _download_to_tmp_and_move(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1915, in _download_to_tmp_and_move
+    http_get(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 558, in http_get
+    return http_get(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/tqdm/std.py", line 1140, in __exit__
+    self.close()
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/tqdm/std.py", line 1290, in close
+    fp_write('')
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/tqdm/std.py", line 1287, in fp_write
+    self.fp.write(str(s))
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/tqdm/utils.py", line 196, in inner
+    return func(*args, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/wandb/sdk/lib/redirect.py", line 648, in write
+    cb(data)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/wandb/sdk/wandb_run.py", line 2386, in <lambda>
+    lambda data: self._console_raw_callback("stderr", data),
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/wandb/sdk/wandb_run.py", line 403, in wrapper_fn
+    return func(self, *args, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/wandb/sdk/wandb_run.py", line 1547, in _console_raw_callback
+    self._backend.interface.publish_output_raw(name, data)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/wandb/sdk/interface/interface.py", line 720, in publish_output_raw
+    o.timestamp.GetCurrentTime()
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/google/protobuf/internal/well_known_types.py", line 174, in GetCurrentTime
+    self.FromDatetime(datetime.datetime.utcnow())
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/google/protobuf/internal/well_known_types.py", line 242, in FromDatetime
+    def FromDatetime(self, dt):
+KeyboardInterrupt

wandb/run-20241031_122005-hpmt1tka/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-10-31T16:20:05.731662Z",
+  "args":  [
+    "--perturbation",
+    "reverse_full",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "6",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1753159852032"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241031_122005-hpmt1tka/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":23}}

wandb/run-20241031_122005-hpmt1tka/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2024-10-31T12:20:05.733730139-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-10-31T12:20:05.733742939-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_122005-hpmt1tka/logs/debug-core.log"}
+{"time":"2024-10-31T12:20:05.842258471-04:00","level":"INFO","msg":"created new stream","id":"hpmt1tka"}
+{"time":"2024-10-31T12:20:05.842312131-04:00","level":"INFO","msg":"stream: started","id":"hpmt1tka"}
+{"time":"2024-10-31T12:20:05.842613751-04:00","level":"INFO","msg":"sender: started","stream_id":"hpmt1tka"}
+{"time":"2024-10-31T12:20:05.842585561-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"hpmt1tka"}}
+{"time":"2024-10-31T12:20:05.842560171-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"hpmt1tka"}}
+{"time":"2024-10-31T12:20:06.023200668-04:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-10-31T12:20:29.351287751-04:00","level":"INFO","msg":"stream: closing","id":"hpmt1tka"}
+{"time":"2024-10-31T12:20:29.351320261-04:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-10-31T12:20:29.351851192-04:00","level":"INFO","msg":"Stopped system monitor"}

wandb/run-20241031_122005-hpmt1tka/logs/debug.log

@@ -0,0 +1,27 @@
+2024-10-31 12:20:05,729 INFO    MainThread:557186 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-10-31 12:20:05,729 INFO    MainThread:557186 [wandb_setup.py:_flush():79] Configure stats pid to 557186
+2024-10-31 12:20:05,729 INFO    MainThread:557186 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-10-31 12:20:05,729 INFO    MainThread:557186 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-10-31 12:20:05,729 INFO    MainThread:557186 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-31 12:20:05,729 INFO    MainThread:557186 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-31 12:20:05,729 INFO    MainThread:557186 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-10-31 12:20:05,729 INFO    MainThread:557186 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-31 12:20:05,729 INFO    MainThread:557186 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_122005-hpmt1tka/logs/debug.log
+2024-10-31 12:20:05,729 INFO    MainThread:557186 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_122005-hpmt1tka/logs/debug-internal.log
+2024-10-31 12:20:05,729 INFO    MainThread:557186 [wandb_init.py:init():621] calling init triggers
+2024-10-31 12:20:05,729 INFO    MainThread:557186 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-10-31 12:20:05,729 INFO    MainThread:557186 [wandb_init.py:init():671] starting backend
+2024-10-31 12:20:05,729 INFO    MainThread:557186 [wandb_init.py:init():675] sending inform_init request
+2024-10-31 12:20:05,731 INFO    MainThread:557186 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-31 12:20:05,731 INFO    MainThread:557186 [wandb_init.py:init():688] backend started and connected
+2024-10-31 12:20:05,734 INFO    MainThread:557186 [wandb_init.py:init():783] updated telemetry
+2024-10-31 12:20:05,762 INFO    MainThread:557186 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-10-31 12:20:06,019 INFO    MainThread:557186 [wandb_init.py:init():867] starting run threads in backend
+2024-10-31 12:20:06,126 INFO    MainThread:557186 [wandb_run.py:_console_start():2463] atexit reg
+2024-10-31 12:20:06,126 INFO    MainThread:557186 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-10-31 12:20:06,126 INFO    MainThread:557186 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-10-31 12:20:06,126 INFO    MainThread:557186 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-10-31 12:20:06,128 INFO    MainThread:557186 [wandb_init.py:init():911] run started, returning control to user process
+2024-10-31 12:20:06,128 INFO    MainThread:557186 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_full', 'train_set': '10M', 'batch_size': 3, 'epoch': 6, 'seed': 0, 'lr': 5e-06}
+2024-10-31 12:20:29,351 WARNING MsgRouterThr:557186 [router.py:message_loop():77] message_loop has been closed

wandb/run-20241031_122113-8ldget07/run-8ldget07.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9a76ee528735e90532a6b232e14bbf1365f717d4bf91b5d03ba85925c40fe6b
+size 19249937