Add files using upload-large-folder tool

.gitattributes

@@ -110,3 +110,8 @@ wandb/run-20241105_163248-thalxhcd/run-thalxhcd.wandb filter=lfs diff=lfs merge=
 wandb/run-20241031_002020-q6ot1vz6/run-q6ot1vz6.wandb filter=lfs diff=lfs merge=lfs -text
 wandb/run-20241106_234348-l3eig11b/run-l3eig11b.wandb filter=lfs diff=lfs merge=lfs -text
 wandb/run-20241031_122114-2k9672ya/run-2k9672ya.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241030_013339-dgadwxty/run-dgadwxty.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241101_200535-hnfjoqai/run-hnfjoqai.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241030_013339-s77qk5li/run-s77qk5li.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241101_200535-6xsf0vem/run-6xsf0vem.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241030_011509-zmlu7388/run-zmlu7388.wandb filter=lfs diff=lfs merge=lfs -text

wandb/run-20241030_010305-4ki7693c/files/config.yaml

@@ -0,0 +1,47 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.9.19
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "3":
+                - 13
+                - 23
+                - 55
+            "4": 3.9.19
+            "5": 0.18.5
+            "6": 4.45.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+batch_size:
+    value: 3
+epoch:
+    value: 7
+perturbation:
+    value: reverse_control
+seed:
+    value: 0
+train_set:
+    value: 10M

wandb/run-20241030_010305-4ki7693c/files/output.log

@@ -0,0 +1,4 @@
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 162, in <module>
+    dataset_name = f"babylm_{args.perturbation}_{args.train_zset}_seed{args.seed}"
+AttributeError: 'Namespace' object has no attribute 'train_zset'

wandb/run-20241030_010305-4ki7693c/files/wandb-metadata.json

@@ -0,0 +1,29 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-10-30T05:03:05.639656Z",
+  "args":  [
+    "--perturbation",
+    "reverse_control",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "7",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py"
+}

wandb/run-20241030_010305-4ki7693c/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":0}}

wandb/run-20241030_010305-4ki7693c/logs/debug-internal.log

@@ -0,0 +1,16 @@
+{"time":"2024-10-30T01:03:05.642979784-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-10-30T01:03:05.643001845-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_010305-4ki7693c/logs/debug-core.log"}
+{"time":"2024-10-30T01:03:05.751540951-04:00","level":"INFO","msg":"created new stream","id":"4ki7693c"}
+{"time":"2024-10-30T01:03:05.751579781-04:00","level":"INFO","msg":"stream: started","id":"4ki7693c"}
+{"time":"2024-10-30T01:03:05.751623931-04:00","level":"INFO","msg":"sender: started","stream_id":"4ki7693c"}
+{"time":"2024-10-30T01:03:05.751594441-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"4ki7693c"}}
+{"time":"2024-10-30T01:03:05.751612271-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"4ki7693c"}}
+{"time":"2024-10-30T01:03:05.943856433-04:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-10-30T01:03:06.036999384-04:00","level":"INFO","msg":"stream: closing","id":"4ki7693c"}
+{"time":"2024-10-30T01:03:06.037043194-04:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-10-30T01:03:06.051196751-04:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-10-30T01:03:07.432977083-04:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-10-30T01:03:07.547907803-04:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"4ki7693c"}}
+{"time":"2024-10-30T01:03:07.547982493-04:00","level":"INFO","msg":"sender: closed","stream_id":"4ki7693c"}
+{"time":"2024-10-30T01:03:07.547981023-04:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"4ki7693c"}}
+{"time":"2024-10-30T01:03:07.548091724-04:00","level":"INFO","msg":"stream: closed","id":"4ki7693c"}

wandb/run-20241030_010305-4ki7693c/logs/debug.log

@@ -0,0 +1,27 @@
+2024-10-30 01:03:05,637 INFO    MainThread:320659 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-10-30 01:03:05,638 INFO    MainThread:320659 [wandb_setup.py:_flush():79] Configure stats pid to 320659
+2024-10-30 01:03:05,638 INFO    MainThread:320659 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-10-30 01:03:05,638 INFO    MainThread:320659 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-10-30 01:03:05,638 INFO    MainThread:320659 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-30 01:03:05,638 INFO    MainThread:320659 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-30 01:03:05,638 INFO    MainThread:320659 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-10-30 01:03:05,638 INFO    MainThread:320659 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-30 01:03:05,638 INFO    MainThread:320659 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_010305-4ki7693c/logs/debug.log
+2024-10-30 01:03:05,638 INFO    MainThread:320659 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_010305-4ki7693c/logs/debug-internal.log
+2024-10-30 01:03:05,638 INFO    MainThread:320659 [wandb_init.py:init():621] calling init triggers
+2024-10-30 01:03:05,638 INFO    MainThread:320659 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-10-30 01:03:05,638 INFO    MainThread:320659 [wandb_init.py:init():671] starting backend
+2024-10-30 01:03:05,638 INFO    MainThread:320659 [wandb_init.py:init():675] sending inform_init request
+2024-10-30 01:03:05,639 INFO    MainThread:320659 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-30 01:03:05,639 INFO    MainThread:320659 [wandb_init.py:init():688] backend started and connected
+2024-10-30 01:03:05,641 INFO    MainThread:320659 [wandb_init.py:init():783] updated telemetry
+2024-10-30 01:03:05,664 INFO    MainThread:320659 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-10-30 01:03:05,940 INFO    MainThread:320659 [wandb_init.py:init():867] starting run threads in backend
+2024-10-30 01:03:06,034 INFO    MainThread:320659 [wandb_run.py:_console_start():2463] atexit reg
+2024-10-30 01:03:06,034 INFO    MainThread:320659 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-10-30 01:03:06,034 INFO    MainThread:320659 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-10-30 01:03:06,034 INFO    MainThread:320659 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-10-30 01:03:06,035 INFO    MainThread:320659 [wandb_init.py:init():911] run started, returning control to user process
+2024-10-30 01:03:06,036 INFO    MainThread:320659 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_control', 'train_set': '10M', 'batch_size': 3, 'epoch': 7, 'seed': 0}
+2024-10-30 01:03:06,037 WARNING MsgRouterThr:320659 [router.py:message_loop():77] message_loop has been closed

wandb/run-20241030_011509-zmlu7388/run-zmlu7388.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:040fe78d8ec4b781ed5ef6142b4febcf6226f4d06df169e2ee75ef4a7b3149d1
+size 163840

wandb/run-20241030_013339-dgadwxty/run-dgadwxty.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:589f69288e34d7adbf24b1cbd3ee3d6189d8cd39e1c0aa92d97593459f8c1f11
+size 20119552

wandb/run-20241030_013339-s77qk5li/run-s77qk5li.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c67abff0d802c9d8d917c03111d5b98030ac30597afe446dadeb16459c0339b
+size 20119552

wandb/run-20241030_112852-qf0srieq/files/config.yaml

@@ -0,0 +1,48 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.9.19
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "3":
+                - 2
+                - 13
+                - 23
+                - 55
+            "4": 3.9.19
+            "5": 0.18.5
+            "6": 4.45.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+batch_size:
+    value: 3
+epoch:
+    value: 3
+perturbation:
+    value: reverse_control
+seed:
+    value: 0
+train_set:
+    value: 10M

wandb/run-20241030_112852-qf0srieq/files/output.log

@@ -0,0 +1,17 @@
+model-00001-of-00002.safetensors: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████| 4.97G/4.97G [01:33<00:00, 41.3MB/s]
+Downloading shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [02:08<00:00, 64.25s/it]
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:04<00:00,  2.30s/it]
+Map: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 18140/18140 [00:48<00:00, 372.82 examples/s]
+tokenized_valid: Dataset({
+    features: ['input_ids', 'attention_mask'],
+    num_rows: 600
+})
+/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
+  warnings.warn(
+[2024-10-30 11:31:56,623] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2024-10-30 11:32:04,855] [INFO] [comm.py:652:init_distributed] cdb=None
+Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination
+Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...
+Loading extension module cpu_adam...
+Time to load cpu_adam op: 4.642748594284058 seconds
+wandb: WARNING Fatal error while uploading data. Some run data will not be synced, but it will still be written to disk. Use `wandb sync` at the end of the run to try uploading.

wandb/run-20241030_112852-qf0srieq/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-10-30T15:28:52.808778Z",
+  "args":  [
+    "--perturbation",
+    "reverse_control",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "3",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1710831611904"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241030_112852-qf0srieq/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":23503}}

wandb/run-20241030_112852-qf0srieq/logs/debug-internal.log

@@ -0,0 +1,107 @@
+{"time":"2024-10-30T11:28:52.811130791-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-10-30T11:28:52.811141081-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_112852-qf0srieq/logs/debug-core.log"}
+{"time":"2024-10-30T11:28:52.917206018-04:00","level":"INFO","msg":"created new stream","id":"qf0srieq"}
+{"time":"2024-10-30T11:28:52.917234588-04:00","level":"INFO","msg":"stream: started","id":"qf0srieq"}
+{"time":"2024-10-30T11:28:52.917337019-04:00","level":"INFO","msg":"sender: started","stream_id":"qf0srieq"}
+{"time":"2024-10-30T11:28:52.917290668-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"qf0srieq"}}
+{"time":"2024-10-30T11:28:52.917283768-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"qf0srieq"}}
+{"time":"2024-10-30T11:28:53.127590893-04:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-10-30T14:03:23.466556608-04:00","level":"ERROR","msg":"HTTP error","status":404,"method":"POST","url":"https://api.wandb.ai/files/yaning1001-dartmouth-college/impossible_llm_reverse/qf0srieq/file_stream"}
+{"time":"2024-10-30T14:03:23.470904408-04:00","level":"ERROR+4","msg":"filestream: fatal error: filestream: failed to upload: 404 Not Found path=files/yaning1001-dartmouth-college/impossible_llm_reverse/qf0srieq/file_stream: {\"error\":\"run impossible_llm_reverse/qf0srieq not found while streaming file\"}"}
+{"time":"2024-10-30T18:00:35.968510571-04:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-10-30T18:00:35.984835172-04:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-10-30T18:00:36.01388325-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:00:36.968916927-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":1.010176656,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:00:38.18070286-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:00:43.234030085-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:00:51.514450735-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:01:08.632732116-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:01:36.992239115-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":61.033489373,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:01:43.980932764-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:02:37.012879837-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":121.054135236,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:02:44.036787297-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:03:37.032386632-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":181.07363706,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:03:44.087114894-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:04:37.058210791-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":241.09946743,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:04:44.146732173-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:05:37.08031714-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":301.121564228,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:05:44.202578641-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:06:37.103155122-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":361.14441734,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:06:44.254811138-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:07:37.129370298-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":421.170622677,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:07:44.304687106-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:08:37.187762317-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":481.229015945,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:08:44.353692443-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:09:37.205365399-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":541.246616438,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:09:44.410062874-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:10:35.95923303-04:00","level":"WARN","msg":"sender: taking a long time","seconds":600.000334216,"work":"WorkRecord(*service_go_proto.Record_Telemetry); Control(connection_id:\"127.0.0.1:54966\")"}
+{"time":"2024-10-30T18:10:37.228433012-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":601.269685091,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:10:44.460819633-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:11:37.252924346-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":661.294174825,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:11:44.510745999-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:12:37.279579159-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":721.320828287,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:12:44.561934485-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:13:37.30132052-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":781.342568848,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:13:44.612933316-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:14:37.326717713-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":841.367977471,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:14:44.662885212-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:15:37.352013001-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":901.39327482,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:15:44.719534816-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:16:37.377043784-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":961.418290923,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:16:44.771072869-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:16:44.771190161-04:00","level":"ERROR","msg":"sender: sendConfig:","error":"api: failed sending: POST https://api.wandb.ai/graphql giving up after 21 attempt(s)"}
+{"time":"2024-10-30T18:16:44.771395633-04:00","level":"INFO","msg":"sender: succeeded after taking longer than expected","seconds":968.812598021,"work":"WorkRecord(*service_go_proto.Record_Telemetry); Control(connection_id:\"127.0.0.1:54966\")"}
+{"time":"2024-10-30T18:16:44.827636949-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:16:44.870800532-04:00","level":"ERROR","msg":"HTTP error","status":404,"method":"POST","url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:16:44.870865293-04:00","level":"ERROR","msg":"runfiles: CreateRunFiles returned error: returned error 404 Not Found: {\"errors\":[{\"message\":\"run impossible_llm_reverse/qf0srieq not found during createRunFiles\",\"path\":[\"createRunFiles\"]}],\"data\":{\"createRunFiles\":null}}"}
+{"time":"2024-10-30T18:16:47.272026705-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:16:52.216994148-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:17:01.180898323-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:17:17.315501289-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:17:37.396983698-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":52.62529182,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:17:53.545827597-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:18:37.418983081-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":112.647293623,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:18:53.600763399-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:19:37.437849459-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":172.666171092,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:19:53.656455438-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:20:37.467570843-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":232.695889516,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:20:53.722959862-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:21:37.493223385-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":292.721535378,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:21:53.77677964-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:22:37.516133857-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":352.744444299,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:22:53.831863788-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:23:37.542829635-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":412.771142798,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:23:53.88651383-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:24:37.566591857-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":472.7949116,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:24:53.943522118-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:25:37.588022475-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":532.816332248,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:25:54.011175643-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:26:37.616856963-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":592.845168906,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:26:44.77275561-04:00","level":"WARN","msg":"sender: taking a long time","seconds":600.000793211,"work":"WorkRecord(*service_go_proto.Request_Defer); Control(local:true  always_send:true)"}
+{"time":"2024-10-30T18:26:54.082149884-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:27:37.638631965-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":652.866944558,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:27:54.135251151-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:28:37.658947741-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":712.887259204,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:28:54.198855109-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:29:37.681597392-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":772.909908875,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:29:54.26606069-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:30:37.704016353-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":832.932328576,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:30:54.329584432-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:31:37.730566652-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":892.958878965,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:31:54.417597707-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:32:37.760964501-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":952.989275424,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:32:54.47204599-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:32:54.472190042-04:00","level":"ERROR","msg":"sender: sendConfig:","error":"api: failed sending: POST https://api.wandb.ai/graphql giving up after 21 attempt(s)"}
+{"time":"2024-10-30T18:32:54.472742607-04:00","level":"INFO","msg":"sender: succeeded after taking longer than expected","seconds":969.700851498,"work":"WorkRecord(*service_go_proto.Request_Defer); Control(local:true  always_send:true)"}
+{"time":"2024-10-30T18:32:54.572982549-04:00","level":"ERROR","msg":"HTTP error","status":404,"method":"POST","url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:32:54.5730265-04:00","level":"ERROR","msg":"runfiles: CreateRunFiles returned error: returned error 404 Not Found: {\"errors\":[{\"message\":\"run impossible_llm_reverse/qf0srieq not found during createRunFiles\",\"path\":[\"createRunFiles\"]}],\"data\":{\"createRunFiles\":null}}"}
+{"time":"2024-10-30T18:32:54.659812312-04:00","level":"ERROR","msg":"HTTP error","status":404,"method":"POST","url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:32:54.660095935-04:00","level":"ERROR","msg":"sender: failed to save job artifact: ArtifactSaver.createManifest: returned error 404 Not Found: {\"errors\":[{\"message\":\"failed to find run impossible_llm_reverse/qf0srieq\",\"path\":[\"createArtifactManifest\"]}],\"data\":{\"createArtifactManifest\":null}}"}
+{"time":"2024-10-30T18:32:54.711516723-04:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-10-30T18:32:54.759969425-04:00","level":"ERROR","msg":"HTTP error","status":404,"method":"POST","url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:32:54.760011335-04:00","level":"ERROR","msg":"runfiles: CreateRunFiles returned error: returned error 404 Not Found: {\"errors\":[{\"message\":\"run impossible_llm_reverse/qf0srieq not found during createRunFiles\",\"path\":[\"createRunFiles\"]}],\"data\":{\"createRunFiles\":null}}"}
+{"time":"2024-10-30T18:32:55.719242378-04:00","level":"INFO","msg":"stream: closing","id":"qf0srieq"}
+{"time":"2024-10-30T18:32:55.719270409-04:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"qf0srieq"}}
+{"time":"2024-10-30T18:32:55.719310959-04:00","level":"INFO","msg":"sender: closed","stream_id":"qf0srieq"}
+{"time":"2024-10-30T18:32:55.719298099-04:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"qf0srieq"}}
+{"time":"2024-10-30T18:32:55.71947517-04:00","level":"INFO","msg":"stream: closed","id":"qf0srieq"}

wandb/run-20241030_112852-qf0srieq/logs/debug.log

@@ -0,0 +1,33 @@
+2024-10-30 11:28:52,806 INFO    MainThread:367767 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-10-30 11:28:52,807 INFO    MainThread:367767 [wandb_setup.py:_flush():79] Configure stats pid to 367767
+2024-10-30 11:28:52,807 INFO    MainThread:367767 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-10-30 11:28:52,807 INFO    MainThread:367767 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-10-30 11:28:52,807 INFO    MainThread:367767 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-30 11:28:52,807 INFO    MainThread:367767 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-30 11:28:52,807 INFO    MainThread:367767 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-10-30 11:28:52,807 INFO    MainThread:367767 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-30 11:28:52,807 INFO    MainThread:367767 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_112852-qf0srieq/logs/debug.log
+2024-10-30 11:28:52,807 INFO    MainThread:367767 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_112852-qf0srieq/logs/debug-internal.log
+2024-10-30 11:28:52,807 INFO    MainThread:367767 [wandb_init.py:init():621] calling init triggers
+2024-10-30 11:28:52,807 INFO    MainThread:367767 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-10-30 11:28:52,807 INFO    MainThread:367767 [wandb_init.py:init():671] starting backend
+2024-10-30 11:28:52,807 INFO    MainThread:367767 [wandb_init.py:init():675] sending inform_init request
+2024-10-30 11:28:52,808 INFO    MainThread:367767 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-30 11:28:52,808 INFO    MainThread:367767 [wandb_init.py:init():688] backend started and connected
+2024-10-30 11:28:52,811 INFO    MainThread:367767 [wandb_init.py:init():783] updated telemetry
+2024-10-30 11:28:52,828 INFO    MainThread:367767 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-10-30 11:28:53,124 INFO    MainThread:367767 [wandb_init.py:init():867] starting run threads in backend
+2024-10-30 11:28:53,211 INFO    MainThread:367767 [wandb_run.py:_console_start():2463] atexit reg
+2024-10-30 11:28:53,211 INFO    MainThread:367767 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-10-30 11:28:53,211 INFO    MainThread:367767 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-10-30 11:28:53,211 INFO    MainThread:367767 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-10-30 11:28:53,213 INFO    MainThread:367767 [wandb_init.py:init():911] run started, returning control to user process
+2024-10-30 11:28:53,213 INFO    MainThread:367767 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_control', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0}
+2024-10-30 18:00:35,944 INFO    MainThread:367767 [wandb_run.py:_finish():2158] finishing run yaning1001-dartmouth-college/impossible_llm_reverse/qf0srieq
+2024-10-30 18:00:35,951 INFO    MainThread:367767 [wandb_run.py:_atexit_cleanup():2426] got exitcode: 0
+2024-10-30 18:00:35,951 INFO    MainThread:367767 [wandb_run.py:_restore():2408] restore
+2024-10-30 18:00:35,968 INFO    MainThread:367767 [wandb_run.py:_restore():2414] restore done
+2024-10-30 18:32:55,714 INFO    MainThread:367767 [wandb_run.py:_footer_history_summary_info():3975] rendering history
+2024-10-30 18:32:55,714 INFO    MainThread:367767 [wandb_run.py:_footer_history_summary_info():4007] rendering summary
+2024-10-30 18:32:55,718 INFO    MainThread:367767 [wandb_run.py:_footer_sync_info():3934] logging synced files

wandb/run-20241030_112853-ognjedxv/files/config.yaml

@@ -0,0 +1,48 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.9.19
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "3":
+                - 2
+                - 13
+                - 23
+                - 55
+            "4": 3.9.19
+            "5": 0.18.5
+            "6": 4.45.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+batch_size:
+    value: 3
+epoch:
+    value: 3
+perturbation:
+    value: reverse_control
+seed:
+    value: 0
+train_set:
+    value: 10M

wandb/run-20241030_112853-ognjedxv/files/output.log

@@ -0,0 +1,19 @@
+Downloading shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [02:08<00:00, 64.05s/it]
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:04<00:00,  2.47s/it]
+Map: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 18140/18140 [00:48<00:00, 372.44 examples/s]
+tokenized_valid: Dataset({
+    features: ['input_ids', 'attention_mask'],
+    num_rows: 600
+})
+/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
+  warnings.warn(
+[2024-10-30 11:31:56,739] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2024-10-30 11:32:04,178] [INFO] [comm.py:652:init_distributed] cdb=None
+Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination
+Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...
+Emitting ninja build file /home/chunhui/.cache/torch_extensions/py39_cu117/cpu_adam/build.ninja...
+Building extension module cpu_adam...
+Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N)
+Loading extension module cpu_adam...
+Time to load cpu_adam op: 4.685936212539673 seconds
+wandb: WARNING Fatal error while uploading data. Some run data will not be synced, but it will still be written to disk. Use `wandb sync` at the end of the run to try uploading.

wandb/run-20241030_112853-ognjedxv/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-10-30T15:28:53.133377Z",
+  "args":  [
+    "--perturbation",
+    "reverse_control",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "3",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1710831611904"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241030_112853-ognjedxv/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":23502}}

wandb/run-20241030_112853-ognjedxv/logs/debug-internal.log

@@ -0,0 +1,108 @@
+{"time":"2024-10-30T11:28:53.136675238-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-10-30T11:28:53.136701068-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_112853-ognjedxv/logs/debug-core.log"}
+{"time":"2024-10-30T11:28:53.347633316-04:00","level":"INFO","msg":"created new stream","id":"ognjedxv"}
+{"time":"2024-10-30T11:28:53.347690336-04:00","level":"INFO","msg":"stream: started","id":"ognjedxv"}
+{"time":"2024-10-30T11:28:53.347800017-04:00","level":"INFO","msg":"sender: started","stream_id":"ognjedxv"}
+{"time":"2024-10-30T11:28:53.347704066-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"ognjedxv"}}
+{"time":"2024-10-30T11:28:53.347733556-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"ognjedxv"}}
+{"time":"2024-10-30T11:28:53.523190791-04:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-10-30T14:02:53.936256906-04:00","level":"ERROR","msg":"HTTP error","status":404,"method":"POST","url":"https://api.wandb.ai/files/yaning1001-dartmouth-college/impossible_llm_reverse/ognjedxv/file_stream"}
+{"time":"2024-10-30T14:02:54.037928715-04:00","level":"ERROR+4","msg":"filestream: fatal error: filestream: failed to upload: 404 Not Found path=files/yaning1001-dartmouth-college/impossible_llm_reverse/ognjedxv/file_stream: {\"error\":\"run impossible_llm_reverse/ognjedxv not found while streaming file\"}"}
+{"time":"2024-10-30T15:32:59.30659372-04:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-10-30T18:00:35.968921574-04:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-10-30T18:00:35.984890772-04:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-10-30T18:00:36.012002318-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:00:36.969753173-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":1.014362711,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:00:38.325921691-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:00:42.497060545-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:00:51.61297613-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:01:09.600899944-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:01:36.999896141-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":61.04451063,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:01:45.006064613-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:02:37.029240457-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":121.073848476,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:02:45.056317872-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:03:37.053072349-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":181.097685018,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:03:45.107280763-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:04:37.077388538-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":241.121996887,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:04:45.166717481-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:05:37.099223886-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":301.143834105,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:05:45.21766182-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:06:37.12482984-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":361.169437169,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:06:45.278269332-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:07:37.142898291-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":421.187507639,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:07:45.329370113-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:08:37.165565448-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":481.210170617,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:08:45.38725225-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:09:37.189610381-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":541.23421759,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:09:45.4390075-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:10:35.955447357-04:00","level":"WARN","msg":"sender: taking a long time","seconds":600.000042295,"work":"WorkRecord(*service_go_proto.Record_Telemetry); Control(connection_id:\"127.0.0.1:47734\")"}
+{"time":"2024-10-30T18:10:37.221958924-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":601.266562773,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:10:45.492534001-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:11:37.252893706-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":661.297499824,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:11:45.544448469-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:12:37.363875954-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":721.408484843,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:12:45.6051509-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:13:37.383426633-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":781.428030622,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:13:45.658741606-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:14:37.40420376-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":841.448812198,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:14:45.714819585-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:15:37.426200511-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":901.47080191,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:15:45.782516393-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:16:37.446643168-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":961.491247447,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:16:45.83431873-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:16:45.834418971-04:00","level":"ERROR","msg":"sender: sendConfig:","error":"api: failed sending: POST https://api.wandb.ai/graphql giving up after 21 attempt(s)"}
+{"time":"2024-10-30T18:16:45.834565243-04:00","level":"INFO","msg":"sender: succeeded after taking longer than expected","seconds":969.879223162,"work":"WorkRecord(*service_go_proto.Record_Telemetry); Control(connection_id:\"127.0.0.1:47734\")"}
+{"time":"2024-10-30T18:16:45.893720374-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:16:45.937388384-04:00","level":"ERROR","msg":"HTTP error","status":404,"method":"POST","url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:16:45.937441754-04:00","level":"ERROR","msg":"runfiles: CreateRunFiles returned error: returned error 404 Not Found: {\"errors\":[{\"message\":\"run impossible_llm_reverse/ognjedxv not found during createRunFiles\",\"path\":[\"createRunFiles\"]}],\"data\":{\"createRunFiles\":null}}"}
+{"time":"2024-10-30T18:16:48.137783109-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:16:52.444026331-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:17:01.754112994-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:17:19.575906609-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:17:37.465079512-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":51.630157695,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:17:53.783058228-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:18:37.485068435-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":111.650150518,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:18:53.833207383-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:19:37.503426143-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":171.668504316,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:19:53.883403259-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:20:37.529190853-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":231.694274566,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:20:53.939024555-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:21:37.552685095-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":291.717760967,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:21:53.995972735-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:22:37.573860393-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":351.738938456,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:22:54.048193661-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:23:37.59771863-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":411.762794442,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:23:54.10633799-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:24:37.618320459-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":471.783396942,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:24:54.157197254-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:25:37.637366385-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":531.802443028,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:25:54.214564623-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:26:37.657934536-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":591.823008329,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:26:45.835568653-04:00","level":"WARN","msg":"sender: taking a long time","seconds":600.000301462,"work":"WorkRecord(*service_go_proto.Request_Defer); Control(local:true  always_send:true)"}
+{"time":"2024-10-30T18:26:54.269308808-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:27:37.67735044-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":651.842428013,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:27:54.325073744-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:28:37.695600522-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":711.860675855,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:28:54.377526276-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:29:37.71622571-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":771.881304503,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:29:54.433826357-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:30:37.735499119-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":831.900579192,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:30:54.489546782-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:31:37.758295272-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":891.923373515,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:31:54.55033287-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:32:37.784133118-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":951.949222191,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:32:54.601982687-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:32:54.602027358-04:00","level":"ERROR","msg":"sender: sendConfig:","error":"api: failed sending: POST https://api.wandb.ai/graphql giving up after 21 attempt(s)"}
+{"time":"2024-10-30T18:32:54.60225771-04:00","level":"INFO","msg":"sender: succeeded after taking longer than expected","seconds":968.76710798,"work":"WorkRecord(*service_go_proto.Request_Defer); Control(local:true  always_send:true)"}
+{"time":"2024-10-30T18:32:54.704245558-04:00","level":"ERROR","msg":"HTTP error","status":404,"method":"POST","url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:32:54.704274299-04:00","level":"ERROR","msg":"runfiles: CreateRunFiles returned error: returned error 404 Not Found: {\"errors\":[{\"message\":\"run impossible_llm_reverse/ognjedxv not found during createRunFiles\",\"path\":[\"createRunFiles\"]}],\"data\":{\"createRunFiles\":null}}"}
+{"time":"2024-10-30T18:32:54.828523016-04:00","level":"ERROR","msg":"HTTP error","status":404,"method":"POST","url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:32:54.828680797-04:00","level":"ERROR","msg":"sender: failed to save job artifact: ArtifactSaver.createManifest: returned error 404 Not Found: {\"errors\":[{\"message\":\"failed to find run impossible_llm_reverse/ognjedxv\",\"path\":[\"createArtifactManifest\"]}],\"data\":{\"createArtifactManifest\":null}}"}
+{"time":"2024-10-30T18:32:54.879693521-04:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-10-30T18:32:54.926933592-04:00","level":"ERROR","msg":"HTTP error","status":404,"method":"POST","url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:32:54.926962902-04:00","level":"ERROR","msg":"runfiles: CreateRunFiles returned error: returned error 404 Not Found: {\"errors\":[{\"message\":\"run impossible_llm_reverse/ognjedxv not found during createRunFiles\",\"path\":[\"createRunFiles\"]}],\"data\":{\"createRunFiles\":null}}"}
+{"time":"2024-10-30T18:32:55.890003709-04:00","level":"INFO","msg":"stream: closing","id":"ognjedxv"}
+{"time":"2024-10-30T18:32:55.89003529-04:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"ognjedxv"}}
+{"time":"2024-10-30T18:32:55.89005759-04:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"ognjedxv"}}
+{"time":"2024-10-30T18:32:55.89010566-04:00","level":"INFO","msg":"sender: closed","stream_id":"ognjedxv"}
+{"time":"2024-10-30T18:32:55.8901257-04:00","level":"INFO","msg":"stream: closed","id":"ognjedxv"}

wandb/run-20241030_112853-ognjedxv/logs/debug.log

@@ -0,0 +1,33 @@
+2024-10-30 11:28:53,129 INFO    MainThread:367765 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-10-30 11:28:53,130 INFO    MainThread:367765 [wandb_setup.py:_flush():79] Configure stats pid to 367765
+2024-10-30 11:28:53,130 INFO    MainThread:367765 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-10-30 11:28:53,130 INFO    MainThread:367765 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-10-30 11:28:53,130 INFO    MainThread:367765 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-30 11:28:53,130 INFO    MainThread:367765 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-30 11:28:53,130 INFO    MainThread:367765 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-10-30 11:28:53,130 INFO    MainThread:367765 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-30 11:28:53,130 INFO    MainThread:367765 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_112853-ognjedxv/logs/debug.log
+2024-10-30 11:28:53,130 INFO    MainThread:367765 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_112853-ognjedxv/logs/debug-internal.log
+2024-10-30 11:28:53,130 INFO    MainThread:367765 [wandb_init.py:init():621] calling init triggers
+2024-10-30 11:28:53,130 INFO    MainThread:367765 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-10-30 11:28:53,130 INFO    MainThread:367765 [wandb_init.py:init():671] starting backend
+2024-10-30 11:28:53,130 INFO    MainThread:367765 [wandb_init.py:init():675] sending inform_init request
+2024-10-30 11:28:53,132 INFO    MainThread:367765 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-30 11:28:53,133 INFO    MainThread:367765 [wandb_init.py:init():688] backend started and connected
+2024-10-30 11:28:53,136 INFO    MainThread:367765 [wandb_init.py:init():783] updated telemetry
+2024-10-30 11:28:53,181 INFO    MainThread:367765 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-10-30 11:28:53,520 INFO    MainThread:367765 [wandb_init.py:init():867] starting run threads in backend
+2024-10-30 11:28:53,609 INFO    MainThread:367765 [wandb_run.py:_console_start():2463] atexit reg
+2024-10-30 11:28:53,609 INFO    MainThread:367765 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-10-30 11:28:53,609 INFO    MainThread:367765 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-10-30 11:28:53,609 INFO    MainThread:367765 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-10-30 11:28:53,610 INFO    MainThread:367765 [wandb_init.py:init():911] run started, returning control to user process
+2024-10-30 11:28:53,611 INFO    MainThread:367765 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_control', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0}
+2024-10-30 18:00:35,945 INFO    MainThread:367765 [wandb_run.py:_finish():2158] finishing run yaning1001-dartmouth-college/impossible_llm_reverse/ognjedxv
+2024-10-30 18:00:35,955 INFO    MainThread:367765 [wandb_run.py:_atexit_cleanup():2426] got exitcode: 0
+2024-10-30 18:00:35,955 INFO    MainThread:367765 [wandb_run.py:_restore():2408] restore
+2024-10-30 18:00:35,968 INFO    MainThread:367765 [wandb_run.py:_restore():2414] restore done
+2024-10-30 18:32:55,882 INFO    MainThread:367765 [wandb_run.py:_footer_history_summary_info():3975] rendering history
+2024-10-30 18:32:55,883 INFO    MainThread:367765 [wandb_run.py:_footer_history_summary_info():4007] rendering summary
+2024-10-30 18:32:55,889 INFO    MainThread:367765 [wandb_run.py:_footer_sync_info():3934] logging synced files

wandb/run-20241030_233740-anh3ext7/files/output.log

@@ -0,0 +1,16 @@
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:06<00:00,  3.01s/it]
+tokenized_valid: Dataset({
+    features: ['input_ids', 'attention_mask'],
+    num_rows: 600
+})
+/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
+  warnings.warn(
+[2024-10-30 23:37:50,877] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2024-10-30 23:37:58,570] [INFO] [comm.py:652:init_distributed] cdb=None
+Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination
+Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...
+Emitting ninja build file /home/chunhui/.cache/torch_extensions/py39_cu117/cpu_adam/build.ninja...
+Building extension module cpu_adam...
+Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N)
+Loading extension module cpu_adam...
+Time to load cpu_adam op: 4.623389482498169 seconds

wandb/run-20241030_233740-anh3ext7/files/requirements.txt

@@ -0,0 +1,147 @@
+funcsigs==1.0.2
+sentry-sdk==2.17.0
+multiprocess==0.70.16
+numpy==1.26.2
+pluralizer==1.2.0
+debugpy==1.6.7
+nvidia-cudnn-cu11==8.5.0.96
+deepspeed==0.15.2
+data==0.4
+pandas==2.1.3
+tomli==2.0.1
+charset-normalizer==3.3.2
+attrs==24.2.0
+aiosignal==1.3.1
+fsspec==2023.10.0
+nvidia-cusparse-cu11==11.7.4.91
+zipp==3.12.0
+mypy-extensions==1.0.0
+datasets==3.0.1
+joblib==1.3.2
+hjson==3.1.0
+traitlets==5.7.1
+stack-data==0.6.0
+transformers==4.45.1
+sympy==1.11.1
+Pygments==2.15.0
+docker-pycreds==0.4.0
+dill==0.3.8
+wheel==0.44.0
+prompt-toolkit==3.0.30
+parso==0.8.3
+ipykernel==6.23.1
+pyarrow==17.0.0
+certifi==2023.11.17
+nvidia-cufft-cu11==10.9.0.58
+six==1.16.0
+pydantic==2.9.2
+click==8.1.7
+nest-asyncio==1.5.6
+gmpy2==2.1.0
+matplotlib==3.8.2
+scipy==1.11.4
+typing_extensions==4.12.2
+statsmodels==0.14.0
+huggingface-hub==0.25.0
+frozenlist==1.4.1
+gpustat==1.1.1
+nvidia-nvtx-cu11==11.7.91
+safetensors==0.4.5
+stanza==1.9.2
+decorator==5.1.1
+seaborn==0.13.0
+sentencepiece==0.2.0
+PyYAML==6.0.1
+black==24.8.0
+protobuf==4.25.1
+pickleshare==0.7.5
+peft==0.13.0
+triton==2.0.0
+nvidia-cuda-runtime-cu11==11.7.99
+Jinja2==3.1.2
+nvidia-cusolver-cu11==11.4.0.1
+executing==1.2.0
+jupyter_client==8.1.0
+pluggy==1.3.0
+cmake==3.30.3
+pytz==2023.3.post1
+aiohappyeyeballs==2.4.2
+kiwisolver==1.4.5
+py-cpuinfo==9.0.0
+Pillow==10.1.0
+ptyprocess==0.7.0
+importlib_resources==6.4.5
+GitPython==3.1.43
+importlib-metadata==6.0.0
+iniconfig==2.0.0
+scikit-learn==1.3.2
+exceptiongroup==1.1.0
+networkx==2.8.6
+accelerate==1.0.0
+nltk==3.8.1
+shutilwhich==1.1.0
+fonttools==4.45.1
+future==0.18.3
+aiohttp==3.10.6
+wcwidth==0.2.5
+idna==3.6
+filelock==3.12.2
+pathspec==0.12.1
+jupyter_core==5.1.0
+lit==18.1.8
+nvidia-curand-cu11==10.2.10.91
+nvidia-cublas-cu11==11.10.3.66
+nvidia-ml-py==12.560.30
+msgpack==1.1.0
+python-dateutil==2.8.2
+blessed==1.20.0
+packaging==23.0
+gitdb==4.0.11
+yarl==1.13.0
+emoji==2.8.0
+tzdata==2023.3
+cycler==0.12.1
+tornado==6.2
+backcall==0.2.0
+plotnine==0.12.4
+ninja==1.11.1.1
+latex==0.7.0
+wandb==0.18.5
+setproctitle==1.3.3
+threadpoolctl==3.2.0
+requests==2.32.3
+pyparsing==3.1.1
+smmap==5.0.1
+pyzmq==23.0.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+matplotlib-inline==0.1.6
+latexcodec==1.0.0
+ipython==8.0.0
+patsy==0.5.3
+contourpy==1.2.0
+multidict==6.1.0
+mizani==0.9.3
+urllib3==2.1.0
+tokenizers==0.20.0
+MarkupSafe==2.1.2
+pip==24.2
+pexpect==4.8.0
+tqdm==4.66.5
+jedi==0.18.2
+pydantic_core==2.23.4
+tempdir==0.7.1
+mpmath==1.2.1
+setuptools==72.1.0
+pytest==7.4.3
+pure-eval==0.2.2
+psutil==5.9.1
+comm==0.1.2
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+regex==2023.10.3
+platformdirs==2.5.2
+asttokens==2.2.1
+torch==2.0.0
+nvidia-nccl-cu11==2.14.3
+xxhash==3.5.0

wandb/run-20241030_233740-anh3ext7/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-10-31T03:37:40.846508Z",
+  "args":  [
+    "--perturbation",
+    "reverse_control",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "3",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1711065919488"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241030_233740-anh3ext7/logs/debug-internal.log

@@ -0,0 +1,8 @@
+{"time":"2024-10-30T23:37:40.850081823-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-10-30T23:37:40.850115643-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_233740-anh3ext7/logs/debug-core.log"}
+{"time":"2024-10-30T23:37:40.962577204-04:00","level":"INFO","msg":"created new stream","id":"anh3ext7"}
+{"time":"2024-10-30T23:37:40.962609945-04:00","level":"INFO","msg":"stream: started","id":"anh3ext7"}
+{"time":"2024-10-30T23:37:40.962627565-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"anh3ext7"}}
+{"time":"2024-10-30T23:37:40.962685265-04:00","level":"INFO","msg":"sender: started","stream_id":"anh3ext7"}
+{"time":"2024-10-30T23:37:40.962685215-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"anh3ext7"}}
+{"time":"2024-10-30T23:37:41.412681383-04:00","level":"INFO","msg":"Starting system monitor"}

wandb/run-20241030_233740-anh3ext7/logs/debug.log

@@ -0,0 +1,26 @@
+2024-10-30 23:37:40,844 INFO    MainThread:464535 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-10-30 23:37:40,844 INFO    MainThread:464535 [wandb_setup.py:_flush():79] Configure stats pid to 464535
+2024-10-30 23:37:40,844 INFO    MainThread:464535 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-10-30 23:37:40,844 INFO    MainThread:464535 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-10-30 23:37:40,844 INFO    MainThread:464535 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-30 23:37:40,844 INFO    MainThread:464535 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-30 23:37:40,844 INFO    MainThread:464535 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-10-30 23:37:40,844 INFO    MainThread:464535 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-30 23:37:40,845 INFO    MainThread:464535 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_233740-anh3ext7/logs/debug.log
+2024-10-30 23:37:40,845 INFO    MainThread:464535 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_233740-anh3ext7/logs/debug-internal.log
+2024-10-30 23:37:40,845 INFO    MainThread:464535 [wandb_init.py:init():621] calling init triggers
+2024-10-30 23:37:40,845 INFO    MainThread:464535 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-10-30 23:37:40,845 INFO    MainThread:464535 [wandb_init.py:init():671] starting backend
+2024-10-30 23:37:40,845 INFO    MainThread:464535 [wandb_init.py:init():675] sending inform_init request
+2024-10-30 23:37:40,846 INFO    MainThread:464535 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-30 23:37:40,846 INFO    MainThread:464535 [wandb_init.py:init():688] backend started and connected
+2024-10-30 23:37:40,849 INFO    MainThread:464535 [wandb_init.py:init():783] updated telemetry
+2024-10-30 23:37:40,879 INFO    MainThread:464535 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-10-30 23:37:41,409 INFO    MainThread:464535 [wandb_init.py:init():867] starting run threads in backend
+2024-10-30 23:37:42,836 INFO    MainThread:464535 [wandb_run.py:_console_start():2463] atexit reg
+2024-10-30 23:37:42,837 INFO    MainThread:464535 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-10-30 23:37:42,837 INFO    MainThread:464535 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-10-30 23:37:42,837 INFO    MainThread:464535 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-10-30 23:37:42,860 INFO    MainThread:464535 [wandb_init.py:init():911] run started, returning control to user process
+2024-10-30 23:37:42,861 INFO    MainThread:464535 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_control', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0}

wandb/run-20241031_114700-78zg7gu4/files/output.log

@@ -0,0 +1,48 @@
+Downloading shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [02:32<00:00, 76.42s/it]
+Loading checkpoint shards: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:04<00:00,  2.42s/it]
+tokenized_valid: Dataset({
+    features: ['input_ids', 'attention_mask'],
+    num_rows: 600
+})
+/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
+  warnings.warn(
+[2024-10-31 11:49:40,267] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2024-10-31 11:49:48,657] [INFO] [comm.py:652:init_distributed] cdb=None
+Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination
+Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...
+Emitting ninja build file /home/chunhui/.cache/torch_extensions/py39_cu117/cpu_adam/build.ninja...
+Building extension module cpu_adam...
+Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N)
+Loading extension module cpu_adam...
+Time to load cpu_adam op: 4.85258412361145 seconds
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 220, in <module>
+    trainer.train()
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/trainer.py", line 2052, in train
+    return inner_training_loop(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/trainer.py", line 2388, in _inner_training_loop
+    tr_loss_step = self.training_step(model, inputs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/trainer.py", line 3518, in training_step
+    self.accelerator.backward(loss, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/accelerate/accelerator.py", line 2238, in backward
+    self.deepspeed_engine_wrapped.backward(loss, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/accelerate/utils/deepspeed.py", line 186, in backward
+    self.engine.backward(loss, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/utils/nvtx.py", line 18, in wrapped_fn
+    ret_val = func(*args, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/runtime/engine.py", line 2020, in backward
+    self.optimizer.backward(loss, retain_graph=retain_graph)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/runtime/zero/stage_1_and_2.py", line 2063, in backward
+    self.loss_scaler.backward(loss.float(), retain_graph=retain_graph)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/runtime/fp16/loss_scaler.py", line 63, in backward
+    scaled_loss.backward(retain_graph=retain_graph)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/torch/_tensor.py", line 487, in backward
+    torch.autograd.backward(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/torch/autograd/__init__.py", line 200, in backward
+    Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
+KeyboardInterrupt
+Error in atexit._run_exitfuncs:
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/deepspeed/ops/transformer/inference/triton/matmul_ext.py", line 27, in is_nfs_path
+    output = subprocess.check_output(['df', '-T', path], encoding='utf-8')
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/subprocess.py", line 424, in check_output

wandb/run-20241031_114700-78zg7gu4/files/requirements.txt

@@ -0,0 +1,147 @@
+funcsigs==1.0.2
+sentry-sdk==2.17.0
+multiprocess==0.70.16
+numpy==1.26.2
+pluralizer==1.2.0
+debugpy==1.6.7
+nvidia-cudnn-cu11==8.5.0.96
+deepspeed==0.15.2
+data==0.4
+pandas==2.1.3
+tomli==2.0.1
+charset-normalizer==3.3.2
+attrs==24.2.0
+aiosignal==1.3.1
+fsspec==2023.10.0
+nvidia-cusparse-cu11==11.7.4.91
+zipp==3.12.0
+mypy-extensions==1.0.0
+datasets==3.0.1
+joblib==1.3.2
+hjson==3.1.0
+traitlets==5.7.1
+stack-data==0.6.0
+transformers==4.45.1
+sympy==1.11.1
+Pygments==2.15.0
+docker-pycreds==0.4.0
+dill==0.3.8
+wheel==0.44.0
+prompt-toolkit==3.0.30
+parso==0.8.3
+ipykernel==6.23.1
+pyarrow==17.0.0
+certifi==2023.11.17
+nvidia-cufft-cu11==10.9.0.58
+six==1.16.0
+pydantic==2.9.2
+click==8.1.7
+nest-asyncio==1.5.6
+gmpy2==2.1.0
+matplotlib==3.8.2
+scipy==1.11.4
+typing_extensions==4.12.2
+statsmodels==0.14.0
+huggingface-hub==0.25.0
+frozenlist==1.4.1
+gpustat==1.1.1
+nvidia-nvtx-cu11==11.7.91
+safetensors==0.4.5
+stanza==1.9.2
+decorator==5.1.1
+seaborn==0.13.0
+sentencepiece==0.2.0
+PyYAML==6.0.1
+black==24.8.0
+protobuf==4.25.1
+pickleshare==0.7.5
+peft==0.13.0
+triton==2.0.0
+nvidia-cuda-runtime-cu11==11.7.99
+Jinja2==3.1.2
+nvidia-cusolver-cu11==11.4.0.1
+executing==1.2.0
+jupyter_client==8.1.0
+pluggy==1.3.0
+cmake==3.30.3
+pytz==2023.3.post1
+aiohappyeyeballs==2.4.2
+kiwisolver==1.4.5
+py-cpuinfo==9.0.0
+Pillow==10.1.0
+ptyprocess==0.7.0
+importlib_resources==6.4.5
+GitPython==3.1.43
+importlib-metadata==6.0.0
+iniconfig==2.0.0
+scikit-learn==1.3.2
+exceptiongroup==1.1.0
+networkx==2.8.6
+accelerate==1.0.0
+nltk==3.8.1
+shutilwhich==1.1.0
+fonttools==4.45.1
+future==0.18.3
+aiohttp==3.10.6
+wcwidth==0.2.5
+idna==3.6
+filelock==3.12.2
+pathspec==0.12.1
+jupyter_core==5.1.0
+lit==18.1.8
+nvidia-curand-cu11==10.2.10.91
+nvidia-cublas-cu11==11.10.3.66
+nvidia-ml-py==12.560.30
+msgpack==1.1.0
+python-dateutil==2.8.2
+blessed==1.20.0
+packaging==23.0
+gitdb==4.0.11
+yarl==1.13.0
+emoji==2.8.0
+tzdata==2023.3
+cycler==0.12.1
+tornado==6.2
+backcall==0.2.0
+plotnine==0.12.4
+ninja==1.11.1.1
+latex==0.7.0
+wandb==0.18.5
+setproctitle==1.3.3
+threadpoolctl==3.2.0
+requests==2.32.3
+pyparsing==3.1.1
+smmap==5.0.1
+pyzmq==23.0.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+matplotlib-inline==0.1.6
+latexcodec==1.0.0
+ipython==8.0.0
+patsy==0.5.3
+contourpy==1.2.0
+multidict==6.1.0
+mizani==0.9.3
+urllib3==2.1.0
+tokenizers==0.20.0
+MarkupSafe==2.1.2
+pip==24.2
+pexpect==4.8.0
+tqdm==4.66.5
+jedi==0.18.2
+pydantic_core==2.23.4
+tempdir==0.7.1
+mpmath==1.2.1
+setuptools==72.1.0
+pytest==7.4.3
+pure-eval==0.2.2
+psutil==5.9.1
+comm==0.1.2
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+regex==2023.10.3
+platformdirs==2.5.2
+asttokens==2.2.1
+torch==2.0.0
+nvidia-nccl-cu11==2.14.3
+xxhash==3.5.0

wandb/run-20241031_114700-78zg7gu4/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-10-31T15:47:00.200293Z",
+  "args":  [
+    "--perturbation",
+    "reverse_full",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "6",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1753158594560"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241031_114700-78zg7gu4/logs/debug-internal.log

@@ -0,0 +1,8 @@
+{"time":"2024-10-31T11:47:00.202347389-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-10-31T11:47:00.202360349-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_114700-78zg7gu4/logs/debug-core.log"}
+{"time":"2024-10-31T11:47:00.312950526-04:00","level":"INFO","msg":"created new stream","id":"78zg7gu4"}
+{"time":"2024-10-31T11:47:00.312997696-04:00","level":"INFO","msg":"stream: started","id":"78zg7gu4"}
+{"time":"2024-10-31T11:47:00.313037447-04:00","level":"INFO","msg":"sender: started","stream_id":"78zg7gu4"}
+{"time":"2024-10-31T11:47:00.313009646-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"78zg7gu4"}}
+{"time":"2024-10-31T11:47:00.313032707-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"78zg7gu4"}}
+{"time":"2024-10-31T11:47:00.524489984-04:00","level":"INFO","msg":"Starting system monitor"}

wandb/run-20241031_114700-78zg7gu4/logs/debug.log

@@ -0,0 +1,26 @@
+2024-10-31 11:47:00,198 INFO    MainThread:554148 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-10-31 11:47:00,198 INFO    MainThread:554148 [wandb_setup.py:_flush():79] Configure stats pid to 554148
+2024-10-31 11:47:00,198 INFO    MainThread:554148 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-10-31 11:47:00,198 INFO    MainThread:554148 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-10-31 11:47:00,198 INFO    MainThread:554148 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-31 11:47:00,198 INFO    MainThread:554148 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-31 11:47:00,198 INFO    MainThread:554148 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-10-31 11:47:00,198 INFO    MainThread:554148 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-31 11:47:00,198 INFO    MainThread:554148 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_114700-78zg7gu4/logs/debug.log
+2024-10-31 11:47:00,198 INFO    MainThread:554148 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_114700-78zg7gu4/logs/debug-internal.log
+2024-10-31 11:47:00,198 INFO    MainThread:554148 [wandb_init.py:init():621] calling init triggers
+2024-10-31 11:47:00,198 INFO    MainThread:554148 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-10-31 11:47:00,198 INFO    MainThread:554148 [wandb_init.py:init():671] starting backend
+2024-10-31 11:47:00,198 INFO    MainThread:554148 [wandb_init.py:init():675] sending inform_init request
+2024-10-31 11:47:00,199 INFO    MainThread:554148 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-31 11:47:00,200 INFO    MainThread:554148 [wandb_init.py:init():688] backend started and connected
+2024-10-31 11:47:00,203 INFO    MainThread:554148 [wandb_init.py:init():783] updated telemetry
+2024-10-31 11:47:00,240 INFO    MainThread:554148 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-10-31 11:47:00,520 INFO    MainThread:554148 [wandb_init.py:init():867] starting run threads in backend
+2024-10-31 11:47:00,638 INFO    MainThread:554148 [wandb_run.py:_console_start():2463] atexit reg
+2024-10-31 11:47:00,638 INFO    MainThread:554148 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-10-31 11:47:00,638 INFO    MainThread:554148 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-10-31 11:47:00,638 INFO    MainThread:554148 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-10-31 11:47:00,640 INFO    MainThread:554148 [wandb_init.py:init():911] run started, returning control to user process
+2024-10-31 11:47:00,640 INFO    MainThread:554148 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_full', 'train_set': '10M', 'batch_size': 3, 'epoch': 6, 'seed': 0, 'lr': 0.0001}

wandb/run-20241101_094656-v2rxhny6/files/output.log

@@ -0,0 +1,13 @@
+Loading checkpoint shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:05<00:00,  2.73s/it]
+tokenized_valid: Dataset({
+    features: ['input_ids', 'attention_mask'],
+    num_rows: 600
+})
+/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
+  warnings.warn(
+[2024-11-01 09:47:04,115] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2024-11-01 09:47:12,997] [INFO] [comm.py:652:init_distributed] cdb=None
+Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination
+Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...
+Loading extension module cpu_adam...
+Time to load cpu_adam op: 4.672824144363403 seconds

wandb/run-20241101_094656-v2rxhny6/files/requirements.txt

@@ -0,0 +1,147 @@
+funcsigs==1.0.2
+sentry-sdk==2.17.0
+multiprocess==0.70.16
+numpy==1.26.2
+pluralizer==1.2.0
+debugpy==1.6.7
+nvidia-cudnn-cu11==8.5.0.96
+deepspeed==0.15.2
+data==0.4
+pandas==2.1.3
+tomli==2.0.1
+charset-normalizer==3.3.2
+attrs==24.2.0
+aiosignal==1.3.1
+fsspec==2023.10.0
+nvidia-cusparse-cu11==11.7.4.91
+zipp==3.12.0
+mypy-extensions==1.0.0
+datasets==3.0.1
+joblib==1.3.2
+hjson==3.1.0
+traitlets==5.7.1
+stack-data==0.6.0
+transformers==4.45.1
+sympy==1.11.1
+Pygments==2.15.0
+docker-pycreds==0.4.0
+dill==0.3.8
+wheel==0.44.0
+prompt-toolkit==3.0.30
+parso==0.8.3
+ipykernel==6.23.1
+pyarrow==17.0.0
+certifi==2023.11.17
+nvidia-cufft-cu11==10.9.0.58
+six==1.16.0
+pydantic==2.9.2
+click==8.1.7
+nest-asyncio==1.5.6
+gmpy2==2.1.0
+matplotlib==3.8.2
+scipy==1.11.4
+typing_extensions==4.12.2
+statsmodels==0.14.0
+huggingface-hub==0.25.0
+frozenlist==1.4.1
+gpustat==1.1.1
+nvidia-nvtx-cu11==11.7.91
+safetensors==0.4.5
+stanza==1.9.2
+decorator==5.1.1
+seaborn==0.13.0
+sentencepiece==0.2.0
+PyYAML==6.0.1
+black==24.8.0
+protobuf==4.25.1
+pickleshare==0.7.5
+peft==0.13.0
+triton==2.0.0
+nvidia-cuda-runtime-cu11==11.7.99
+Jinja2==3.1.2
+nvidia-cusolver-cu11==11.4.0.1
+executing==1.2.0
+jupyter_client==8.1.0
+pluggy==1.3.0
+cmake==3.30.3
+pytz==2023.3.post1
+aiohappyeyeballs==2.4.2
+kiwisolver==1.4.5
+py-cpuinfo==9.0.0
+Pillow==10.1.0
+ptyprocess==0.7.0
+importlib_resources==6.4.5
+GitPython==3.1.43
+importlib-metadata==6.0.0
+iniconfig==2.0.0
+scikit-learn==1.3.2
+exceptiongroup==1.1.0
+networkx==2.8.6
+accelerate==1.0.0
+nltk==3.8.1
+shutilwhich==1.1.0
+fonttools==4.45.1
+future==0.18.3
+aiohttp==3.10.6
+wcwidth==0.2.5
+idna==3.6
+filelock==3.12.2
+pathspec==0.12.1
+jupyter_core==5.1.0
+lit==18.1.8
+nvidia-curand-cu11==10.2.10.91
+nvidia-cublas-cu11==11.10.3.66
+nvidia-ml-py==12.560.30
+msgpack==1.1.0
+python-dateutil==2.8.2
+blessed==1.20.0
+packaging==23.0
+gitdb==4.0.11
+yarl==1.13.0
+emoji==2.8.0
+tzdata==2023.3
+cycler==0.12.1
+tornado==6.2
+backcall==0.2.0
+plotnine==0.12.4
+ninja==1.11.1.1
+latex==0.7.0
+wandb==0.18.5
+setproctitle==1.3.3
+threadpoolctl==3.2.0
+requests==2.32.3
+pyparsing==3.1.1
+smmap==5.0.1
+pyzmq==23.0.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+matplotlib-inline==0.1.6
+latexcodec==1.0.0
+ipython==8.0.0
+patsy==0.5.3
+contourpy==1.2.0
+multidict==6.1.0
+mizani==0.9.3
+urllib3==2.1.0
+tokenizers==0.20.0
+MarkupSafe==2.1.2
+pip==24.2
+pexpect==4.8.0
+tqdm==4.66.5
+jedi==0.18.2
+pydantic_core==2.23.4
+tempdir==0.7.1
+mpmath==1.2.1
+setuptools==72.1.0
+pytest==7.4.3
+pure-eval==0.2.2
+psutil==5.9.1
+comm==0.1.2
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+regex==2023.10.3
+platformdirs==2.5.2
+asttokens==2.2.1
+torch==2.0.0
+nvidia-nccl-cu11==2.14.3
+xxhash==3.5.0

wandb/run-20241101_094656-v2rxhny6/logs/debug-internal.log

@@ -0,0 +1,8 @@
+{"time":"2024-11-01T09:46:56.281693958-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-11-01T09:46:56.281708498-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_094656-v2rxhny6/logs/debug-core.log"}
+{"time":"2024-11-01T09:46:56.389353319-04:00","level":"INFO","msg":"created new stream","id":"v2rxhny6"}
+{"time":"2024-11-01T09:46:56.3894075-04:00","level":"INFO","msg":"stream: started","id":"v2rxhny6"}
+{"time":"2024-11-01T09:46:56.38944488-04:00","level":"INFO","msg":"sender: started","stream_id":"v2rxhny6"}
+{"time":"2024-11-01T09:46:56.38948828-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"v2rxhny6"}}
+{"time":"2024-11-01T09:46:56.38944628-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"v2rxhny6"}}
+{"time":"2024-11-01T09:46:56.711129595-04:00","level":"INFO","msg":"Starting system monitor"}

wandb/run-20241101_094656-v2rxhny6/logs/debug.log

@@ -0,0 +1,26 @@
+2024-11-01 09:46:56,277 INFO    MainThread:786687 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-11-01 09:46:56,277 INFO    MainThread:786687 [wandb_setup.py:_flush():79] Configure stats pid to 786687
+2024-11-01 09:46:56,277 INFO    MainThread:786687 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-11-01 09:46:56,277 INFO    MainThread:786687 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-11-01 09:46:56,277 INFO    MainThread:786687 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-11-01 09:46:56,277 INFO    MainThread:786687 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-11-01 09:46:56,277 INFO    MainThread:786687 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-11-01 09:46:56,277 INFO    MainThread:786687 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-11-01 09:46:56,277 INFO    MainThread:786687 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_094656-v2rxhny6/logs/debug.log
+2024-11-01 09:46:56,277 INFO    MainThread:786687 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_094656-v2rxhny6/logs/debug-internal.log
+2024-11-01 09:46:56,277 INFO    MainThread:786687 [wandb_init.py:init():621] calling init triggers
+2024-11-01 09:46:56,277 INFO    MainThread:786687 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-11-01 09:46:56,277 INFO    MainThread:786687 [wandb_init.py:init():671] starting backend
+2024-11-01 09:46:56,278 INFO    MainThread:786687 [wandb_init.py:init():675] sending inform_init request
+2024-11-01 09:46:56,279 INFO    MainThread:786687 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-11-01 09:46:56,279 INFO    MainThread:786687 [wandb_init.py:init():688] backend started and connected
+2024-11-01 09:46:56,282 INFO    MainThread:786687 [wandb_init.py:init():783] updated telemetry
+2024-11-01 09:46:56,330 INFO    MainThread:786687 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-11-01 09:46:56,707 INFO    MainThread:786687 [wandb_init.py:init():867] starting run threads in backend
+2024-11-01 09:46:56,815 INFO    MainThread:786687 [wandb_run.py:_console_start():2463] atexit reg
+2024-11-01 09:46:56,815 INFO    MainThread:786687 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-11-01 09:46:56,815 INFO    MainThread:786687 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-11-01 09:46:56,815 INFO    MainThread:786687 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-11-01 09:46:56,817 INFO    MainThread:786687 [wandb_init.py:init():911] run started, returning control to user process
+2024-11-01 09:46:56,818 INFO    MainThread:786687 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_control', 'train_set': '10M', 'batch_size': 3, 'epoch': 7, 'seed': 0, 'lr': 5e-06}

wandb/run-20241101_200535-6xsf0vem/run-6xsf0vem.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d0abd0bb8dcd60a9d99825361276dc45d95ba9a07cabf7f789f5a802d5f115d
+size 131072

wandb/run-20241101_200535-hnfjoqai/run-hnfjoqai.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1d23a55d4b39da70237bf1611118dc5cdd7f3cefeb2d71e7f0a34e03d8a4c7a
+size 131072

wandb/run-20241105_155954-wehwcr47/files/config.yaml

@@ -0,0 +1,49 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.9.19
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "3":
+                - 13
+                - 23
+                - 55
+            "4": 3.9.19
+            "5": 0.18.5
+            "6": 4.45.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+batch_size:
+    value: 3
+epoch:
+    value: 3
+lr:
+    value: 5e-06
+perturbation:
+    value: shuffle_deterministic21
+seed:
+    value: 0
+train_set:
+    value: 10M

wandb/run-20241105_155954-wehwcr47/files/output.log

@@ -0,0 +1,19 @@
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/pathlib.py", line 1323, in mkdir
+    self._accessor.mkdir(self, mode)
+FileNotFoundError: [Errno 2] No such file or directory: '/home/chunhui/.cache/huggingface/datasets/babylm_dataset_test/babylm_shuffle_deterministic21_10M_seed0/0.0.0'
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 165, in <module>
+    dataset = load_dataset('babylm_dataset_test.py', name=dataset_name, trust_remote_code=True)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/load.py", line 2096, in load_dataset
+    builder_instance.download_and_prepare(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/builder.py", line 855, in download_and_prepare
+    Path(self._output_dir).parent.mkdir(parents=True, exist_ok=True)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/pathlib.py", line 1327, in mkdir
+    self.parent.mkdir(parents=True, exist_ok=True)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/pathlib.py", line 1323, in mkdir
+    self._accessor.mkdir(self, mode)
+OSError: [Errno 28] No space left on device: '/home/chunhui/.cache/huggingface/datasets/babylm_dataset_test/babylm_shuffle_deterministic21_10M_seed0'

wandb/run-20241105_155954-wehwcr47/files/requirements.txt

@@ -0,0 +1,147 @@
+funcsigs==1.0.2
+sentry-sdk==2.17.0
+multiprocess==0.70.16
+numpy==1.26.2
+pluralizer==1.2.0
+debugpy==1.6.7
+nvidia-cudnn-cu11==8.5.0.96
+deepspeed==0.15.2
+data==0.4
+pandas==2.1.3
+tomli==2.0.1
+charset-normalizer==3.3.2
+attrs==24.2.0
+aiosignal==1.3.1
+fsspec==2023.10.0
+nvidia-cusparse-cu11==11.7.4.91
+zipp==3.12.0
+mypy-extensions==1.0.0
+datasets==3.0.1
+joblib==1.3.2
+hjson==3.1.0
+traitlets==5.7.1
+stack-data==0.6.0
+transformers==4.45.1
+sympy==1.11.1
+Pygments==2.15.0
+docker-pycreds==0.4.0
+dill==0.3.8
+wheel==0.44.0
+prompt-toolkit==3.0.30
+parso==0.8.3
+ipykernel==6.23.1
+pyarrow==17.0.0
+certifi==2023.11.17
+nvidia-cufft-cu11==10.9.0.58
+six==1.16.0
+pydantic==2.9.2
+click==8.1.7
+nest-asyncio==1.5.6
+gmpy2==2.1.0
+matplotlib==3.8.2
+scipy==1.11.4
+typing_extensions==4.12.2
+statsmodels==0.14.0
+huggingface-hub==0.25.0
+frozenlist==1.4.1
+gpustat==1.1.1
+nvidia-nvtx-cu11==11.7.91
+safetensors==0.4.5
+stanza==1.9.2
+decorator==5.1.1
+seaborn==0.13.0
+sentencepiece==0.2.0
+PyYAML==6.0.1
+black==24.8.0
+protobuf==4.25.1
+pickleshare==0.7.5
+peft==0.13.0
+triton==2.0.0
+nvidia-cuda-runtime-cu11==11.7.99
+Jinja2==3.1.2
+nvidia-cusolver-cu11==11.4.0.1
+executing==1.2.0
+jupyter_client==8.1.0
+pluggy==1.3.0
+cmake==3.30.3
+pytz==2023.3.post1
+aiohappyeyeballs==2.4.2
+kiwisolver==1.4.5
+py-cpuinfo==9.0.0
+Pillow==10.1.0
+ptyprocess==0.7.0
+importlib_resources==6.4.5
+GitPython==3.1.43
+importlib-metadata==6.0.0
+iniconfig==2.0.0
+scikit-learn==1.3.2
+exceptiongroup==1.1.0
+networkx==2.8.6
+accelerate==1.0.0
+nltk==3.8.1
+shutilwhich==1.1.0
+fonttools==4.45.1
+future==0.18.3
+aiohttp==3.10.6
+wcwidth==0.2.5
+idna==3.6
+filelock==3.12.2
+pathspec==0.12.1
+jupyter_core==5.1.0
+lit==18.1.8
+nvidia-curand-cu11==10.2.10.91
+nvidia-cublas-cu11==11.10.3.66
+nvidia-ml-py==12.560.30
+msgpack==1.1.0
+python-dateutil==2.8.2
+blessed==1.20.0
+packaging==23.0
+gitdb==4.0.11
+yarl==1.13.0
+emoji==2.8.0
+tzdata==2023.3
+cycler==0.12.1
+tornado==6.2
+backcall==0.2.0
+plotnine==0.12.4
+ninja==1.11.1.1
+latex==0.7.0
+wandb==0.18.5
+setproctitle==1.3.3
+threadpoolctl==3.2.0
+requests==2.32.3
+pyparsing==3.1.1
+smmap==5.0.1
+pyzmq==23.0.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+matplotlib-inline==0.1.6
+latexcodec==1.0.0
+ipython==8.0.0
+patsy==0.5.3
+contourpy==1.2.0
+multidict==6.1.0
+mizani==0.9.3
+urllib3==2.1.0
+tokenizers==0.20.0
+MarkupSafe==2.1.2
+pip==24.2
+pexpect==4.8.0
+tqdm==4.66.5
+jedi==0.18.2
+pydantic_core==2.23.4
+tempdir==0.7.1
+mpmath==1.2.1
+setuptools==72.1.0
+pytest==7.4.3
+pure-eval==0.2.2
+psutil==5.9.1
+comm==0.1.2
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+regex==2023.10.3
+platformdirs==2.5.2
+asttokens==2.2.1
+torch==2.0.0
+nvidia-nccl-cu11==2.14.3
+xxhash==3.5.0

wandb/run-20241105_155954-wehwcr47/files/wandb-metadata.json

@@ -0,0 +1,44 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-11-05T20:59:54.612366Z",
+  "args":  [
+    "--perturbation",
+    "shuffle_deterministic21",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "3",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1792550322176"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  }
+}

wandb/run-20241105_155954-wehwcr47/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":5}}

wandb/run-20241105_155954-wehwcr47/logs/debug-internal.log

@@ -0,0 +1,17 @@
+{"time":"2024-11-05T15:59:54.614675371-05:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-11-05T15:59:54.614700731-05:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_155954-wehwcr47/logs/debug-core.log"}
+{"time":"2024-11-05T15:59:59.638514173-05:00","level":"INFO","msg":"created new stream","id":"wehwcr47"}
+{"time":"2024-11-05T15:59:59.638580933-05:00","level":"INFO","msg":"stream: started","id":"wehwcr47"}
+{"time":"2024-11-05T15:59:59.638616623-05:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"wehwcr47"}}
+{"time":"2024-11-05T15:59:59.638821764-05:00","level":"INFO","msg":"sender: started","stream_id":"wehwcr47"}
+{"time":"2024-11-05T15:59:59.638739974-05:00","level":"INFO","msg":"handler: started","stream_id":{"value":"wehwcr47"}}
+{"time":"2024-11-05T15:59:59.858951434-05:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-11-05T15:59:59.971734094-05:00","level":"INFO","msg":"stream: closing","id":"wehwcr47"}
+{"time":"2024-11-05T15:59:59.971766184-05:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-11-05T15:59:59.971842264-05:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-11-05T16:00:00.249254784-05:00","level":"ERROR","msg":"sender: sendDefer: failed to build job artifact","error":"failed to write data to file: write /tmp/tmpfile-1264752096: no space left on device"}
+{"time":"2024-11-05T16:00:00.589895964-05:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-11-05T16:00:00.706195627-05:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"wehwcr47"}}
+{"time":"2024-11-05T16:00:00.706225787-05:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"wehwcr47"}}
+{"time":"2024-11-05T16:00:00.706276908-05:00","level":"INFO","msg":"sender: closed","stream_id":"wehwcr47"}
+{"time":"2024-11-05T16:00:00.706283998-05:00","level":"INFO","msg":"stream: closed","id":"wehwcr47"}

wandb/run-20241105_155954-wehwcr47/logs/debug.log

@@ -0,0 +1,27 @@
+2024-11-05 15:59:54,610 INFO    MainThread:1769193 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-11-05 15:59:54,610 INFO    MainThread:1769193 [wandb_setup.py:_flush():79] Configure stats pid to 1769193
+2024-11-05 15:59:54,610 INFO    MainThread:1769193 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-11-05 15:59:54,610 INFO    MainThread:1769193 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-11-05 15:59:54,610 INFO    MainThread:1769193 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-11-05 15:59:54,610 INFO    MainThread:1769193 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-11-05 15:59:54,610 INFO    MainThread:1769193 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-11-05 15:59:54,610 INFO    MainThread:1769193 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-11-05 15:59:54,610 INFO    MainThread:1769193 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_155954-wehwcr47/logs/debug.log
+2024-11-05 15:59:54,610 INFO    MainThread:1769193 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_155954-wehwcr47/logs/debug-internal.log
+2024-11-05 15:59:54,611 INFO    MainThread:1769193 [wandb_init.py:init():621] calling init triggers
+2024-11-05 15:59:54,611 INFO    MainThread:1769193 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-11-05 15:59:54,611 INFO    MainThread:1769193 [wandb_init.py:init():671] starting backend
+2024-11-05 15:59:54,611 INFO    MainThread:1769193 [wandb_init.py:init():675] sending inform_init request
+2024-11-05 15:59:54,612 INFO    MainThread:1769193 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-11-05 15:59:54,612 INFO    MainThread:1769193 [wandb_init.py:init():688] backend started and connected
+2024-11-05 15:59:54,614 INFO    MainThread:1769193 [wandb_init.py:init():783] updated telemetry
+2024-11-05 15:59:54,634 INFO    MainThread:1769193 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-11-05 15:59:59,856 INFO    MainThread:1769193 [wandb_init.py:init():867] starting run threads in backend
+2024-11-05 15:59:59,945 INFO    MainThread:1769193 [wandb_run.py:_console_start():2463] atexit reg
+2024-11-05 15:59:59,945 INFO    MainThread:1769193 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-11-05 15:59:59,946 INFO    MainThread:1769193 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-11-05 15:59:59,946 INFO    MainThread:1769193 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-11-05 15:59:59,947 INFO    MainThread:1769193 [wandb_init.py:init():911] run started, returning control to user process
+2024-11-05 15:59:59,948 INFO    MainThread:1769193 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_deterministic21', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0, 'lr': 5e-06}
+2024-11-05 15:59:59,971 WARNING MsgRouterThr:1769193 [router.py:message_loop():77] message_loop has been closed

wandb/run-20241105_161113-xd1fe9ua/files/config.yaml

@@ -0,0 +1,49 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.9.19
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "3":
+                - 13
+                - 23
+                - 55
+            "4": 3.9.19
+            "5": 0.18.5
+            "6": 4.45.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+batch_size:
+    value: 3
+epoch:
+    value: 3
+lr:
+    value: 5e-06
+perturbation:
+    value: shuffle_deterministic21
+seed:
+    value: 0
+train_set:
+    value: 10M

wandb/run-20241105_161113-xd1fe9ua/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-11-05T21:11:13.777933Z",
+  "args":  [
+    "--perturbation",
+    "shuffle_deterministic21",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "3",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1792542838784"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}