Add files using upload-large-folder tool

.gitattributes

@@ -97,3 +97,7 @@ wandb/run-20241101_202058-hjyig8so/run-hjyig8so.wandb filter=lfs diff=lfs merge=
 wandb/run-20241101_012733-3tsgnm2p/run-3tsgnm2p.wandb filter=lfs diff=lfs merge=lfs -text
 wandb/run-20241030_112852-av3r7rx8/run-av3r7rx8.wandb filter=lfs diff=lfs merge=lfs -text
 wandb/run-20241115_125218-rrve0rbk/run-rrve0rbk.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241129_083813-1lx1o6j4/run-1lx1o6j4.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241101_200535-kh416n4m/run-kh416n4m.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241101_202058-jijqbvs1/run-jijqbvs1.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241129_083813-gsvlu1z8/run-gsvlu1z8.wandb filter=lfs diff=lfs merge=lfs -text

wandb/run-20241030_010759-v4mz2z5o/files/config.yaml

@@ -0,0 +1,47 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.9.19
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "3":
+                - 13
+                - 23
+                - 55
+            "4": 3.9.19
+            "5": 0.18.5
+            "6": 4.45.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+batch_size:
+    value: 3
+epoch:
+    value: 7
+perturbation:
+    value: reverse_control
+seed:
+    value: 0
+train_set:
+    value: 10M

wandb/run-20241030_010759-v4mz2z5o/files/output.log

@@ -0,0 +1,4 @@
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 162, in <module>
+    dataset_name = f"babylm_{args.perturbation}_{args.train_zset}_seed{args.seed}"
+AttributeError: 'Namespace' object has no attribute 'train_zset'

wandb/run-20241030_010759-v4mz2z5o/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-10-30T05:07:59.035692Z",
+  "args":  [
+    "--perturbation",
+    "reverse_control",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "7",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1719200268288"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241030_010759-v4mz2z5o/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":0}}

wandb/run-20241030_010759-v4mz2z5o/logs/debug-internal.log

@@ -0,0 +1,16 @@
+{"time":"2024-10-30T01:07:59.037884269-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-10-30T01:07:59.037895309-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_010759-v4mz2z5o/logs/debug-core.log"}
+{"time":"2024-10-30T01:07:59.144238393-04:00","level":"INFO","msg":"created new stream","id":"v4mz2z5o"}
+{"time":"2024-10-30T01:07:59.144273343-04:00","level":"INFO","msg":"stream: started","id":"v4mz2z5o"}
+{"time":"2024-10-30T01:07:59.144297273-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"v4mz2z5o"}}
+{"time":"2024-10-30T01:07:59.144344244-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"v4mz2z5o"}}
+{"time":"2024-10-30T01:07:59.144382634-04:00","level":"INFO","msg":"sender: started","stream_id":"v4mz2z5o"}
+{"time":"2024-10-30T01:07:59.298984547-04:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-10-30T01:07:59.387117067-04:00","level":"INFO","msg":"stream: closing","id":"v4mz2z5o"}
+{"time":"2024-10-30T01:07:59.387151617-04:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-10-30T01:07:59.387463689-04:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-10-30T01:07:59.920617799-04:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-10-30T01:08:00.033586148-04:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"v4mz2z5o"}}
+{"time":"2024-10-30T01:08:00.033615528-04:00","level":"INFO","msg":"sender: closed","stream_id":"v4mz2z5o"}
+{"time":"2024-10-30T01:08:00.033608968-04:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"v4mz2z5o"}}
+{"time":"2024-10-30T01:08:00.033726869-04:00","level":"INFO","msg":"stream: closed","id":"v4mz2z5o"}

wandb/run-20241030_010759-v4mz2z5o/logs/debug.log

@@ -0,0 +1,27 @@
+2024-10-30 01:07:59,034 INFO    MainThread:322461 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-10-30 01:07:59,034 INFO    MainThread:322461 [wandb_setup.py:_flush():79] Configure stats pid to 322461
+2024-10-30 01:07:59,034 INFO    MainThread:322461 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-10-30 01:07:59,034 INFO    MainThread:322461 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-10-30 01:07:59,034 INFO    MainThread:322461 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-30 01:07:59,034 INFO    MainThread:322461 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-30 01:07:59,034 INFO    MainThread:322461 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-10-30 01:07:59,034 INFO    MainThread:322461 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-30 01:07:59,034 INFO    MainThread:322461 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_010759-v4mz2z5o/logs/debug.log
+2024-10-30 01:07:59,034 INFO    MainThread:322461 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_010759-v4mz2z5o/logs/debug-internal.log
+2024-10-30 01:07:59,034 INFO    MainThread:322461 [wandb_init.py:init():621] calling init triggers
+2024-10-30 01:07:59,034 INFO    MainThread:322461 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-10-30 01:07:59,034 INFO    MainThread:322461 [wandb_init.py:init():671] starting backend
+2024-10-30 01:07:59,034 INFO    MainThread:322461 [wandb_init.py:init():675] sending inform_init request
+2024-10-30 01:07:59,035 INFO    MainThread:322461 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-30 01:07:59,035 INFO    MainThread:322461 [wandb_init.py:init():688] backend started and connected
+2024-10-30 01:07:59,038 INFO    MainThread:322461 [wandb_init.py:init():783] updated telemetry
+2024-10-30 01:07:59,069 INFO    MainThread:322461 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-10-30 01:07:59,295 INFO    MainThread:322461 [wandb_init.py:init():867] starting run threads in backend
+2024-10-30 01:07:59,384 INFO    MainThread:322461 [wandb_run.py:_console_start():2463] atexit reg
+2024-10-30 01:07:59,384 INFO    MainThread:322461 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-10-30 01:07:59,384 INFO    MainThread:322461 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-10-30 01:07:59,384 INFO    MainThread:322461 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-10-30 01:07:59,385 INFO    MainThread:322461 [wandb_init.py:init():911] run started, returning control to user process
+2024-10-30 01:07:59,386 INFO    MainThread:322461 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_control', 'train_set': '10M', 'batch_size': 3, 'epoch': 7, 'seed': 0}
+2024-10-30 01:07:59,387 WARNING MsgRouterThr:322461 [router.py:message_loop():77] message_loop has been closed

wandb/run-20241030_231835-o1t74f3e/files/output.log

@@ -0,0 +1,13 @@
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:05<00:00,  2.78s/it]
+tokenized_valid: Dataset({
+    features: ['input_ids', 'attention_mask'],
+    num_rows: 600
+})
+/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
+  warnings.warn(
+[2024-10-30 23:18:42,945] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2024-10-30 23:18:52,440] [INFO] [comm.py:652:init_distributed] cdb=None
+Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination
+Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...
+Loading extension module cpu_adam...
+Time to load cpu_adam op: 5.8972320556640625 seconds

wandb/run-20241030_231835-o1t74f3e/files/requirements.txt

@@ -0,0 +1,147 @@
+funcsigs==1.0.2
+sentry-sdk==2.17.0
+multiprocess==0.70.16
+numpy==1.26.2
+pluralizer==1.2.0
+debugpy==1.6.7
+nvidia-cudnn-cu11==8.5.0.96
+deepspeed==0.15.2
+data==0.4
+pandas==2.1.3
+tomli==2.0.1
+charset-normalizer==3.3.2
+attrs==24.2.0
+aiosignal==1.3.1
+fsspec==2023.10.0
+nvidia-cusparse-cu11==11.7.4.91
+zipp==3.12.0
+mypy-extensions==1.0.0
+datasets==3.0.1
+joblib==1.3.2
+hjson==3.1.0
+traitlets==5.7.1
+stack-data==0.6.0
+transformers==4.45.1
+sympy==1.11.1
+Pygments==2.15.0
+docker-pycreds==0.4.0
+dill==0.3.8
+wheel==0.44.0
+prompt-toolkit==3.0.30
+parso==0.8.3
+ipykernel==6.23.1
+pyarrow==17.0.0
+certifi==2023.11.17
+nvidia-cufft-cu11==10.9.0.58
+six==1.16.0
+pydantic==2.9.2
+click==8.1.7
+nest-asyncio==1.5.6
+gmpy2==2.1.0
+matplotlib==3.8.2
+scipy==1.11.4
+typing_extensions==4.12.2
+statsmodels==0.14.0
+huggingface-hub==0.25.0
+frozenlist==1.4.1
+gpustat==1.1.1
+nvidia-nvtx-cu11==11.7.91
+safetensors==0.4.5
+stanza==1.9.2
+decorator==5.1.1
+seaborn==0.13.0
+sentencepiece==0.2.0
+PyYAML==6.0.1
+black==24.8.0
+protobuf==4.25.1
+pickleshare==0.7.5
+peft==0.13.0
+triton==2.0.0
+nvidia-cuda-runtime-cu11==11.7.99
+Jinja2==3.1.2
+nvidia-cusolver-cu11==11.4.0.1
+executing==1.2.0
+jupyter_client==8.1.0
+pluggy==1.3.0
+cmake==3.30.3
+pytz==2023.3.post1
+aiohappyeyeballs==2.4.2
+kiwisolver==1.4.5
+py-cpuinfo==9.0.0
+Pillow==10.1.0
+ptyprocess==0.7.0
+importlib_resources==6.4.5
+GitPython==3.1.43
+importlib-metadata==6.0.0
+iniconfig==2.0.0
+scikit-learn==1.3.2
+exceptiongroup==1.1.0
+networkx==2.8.6
+accelerate==1.0.0
+nltk==3.8.1
+shutilwhich==1.1.0
+fonttools==4.45.1
+future==0.18.3
+aiohttp==3.10.6
+wcwidth==0.2.5
+idna==3.6
+filelock==3.12.2
+pathspec==0.12.1
+jupyter_core==5.1.0
+lit==18.1.8
+nvidia-curand-cu11==10.2.10.91
+nvidia-cublas-cu11==11.10.3.66
+nvidia-ml-py==12.560.30
+msgpack==1.1.0
+python-dateutil==2.8.2
+blessed==1.20.0
+packaging==23.0
+gitdb==4.0.11
+yarl==1.13.0
+emoji==2.8.0
+tzdata==2023.3
+cycler==0.12.1
+tornado==6.2
+backcall==0.2.0
+plotnine==0.12.4
+ninja==1.11.1.1
+latex==0.7.0
+wandb==0.18.5
+setproctitle==1.3.3
+threadpoolctl==3.2.0
+requests==2.32.3
+pyparsing==3.1.1
+smmap==5.0.1
+pyzmq==23.0.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+matplotlib-inline==0.1.6
+latexcodec==1.0.0
+ipython==8.0.0
+patsy==0.5.3
+contourpy==1.2.0
+multidict==6.1.0
+mizani==0.9.3
+urllib3==2.1.0
+tokenizers==0.20.0
+MarkupSafe==2.1.2
+pip==24.2
+pexpect==4.8.0
+tqdm==4.66.5
+jedi==0.18.2
+pydantic_core==2.23.4
+tempdir==0.7.1
+mpmath==1.2.1
+setuptools==72.1.0
+pytest==7.4.3
+pure-eval==0.2.2
+psutil==5.9.1
+comm==0.1.2
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+regex==2023.10.3
+platformdirs==2.5.2
+asttokens==2.2.1
+torch==2.0.0
+nvidia-nccl-cu11==2.14.3
+xxhash==3.5.0

wandb/run-20241030_231835-o1t74f3e/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-10-31T03:18:35.151538Z",
+  "args":  [
+    "--perturbation",
+    "reverse_full",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "3",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1711064555520"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241030_231835-o1t74f3e/logs/debug-internal.log

@@ -0,0 +1,8 @@
+{"time":"2024-10-30T23:18:35.153366138-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-10-30T23:18:35.153377718-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_231835-o1t74f3e/logs/debug-core.log"}
+{"time":"2024-10-30T23:18:35.259200227-04:00","level":"INFO","msg":"created new stream","id":"o1t74f3e"}
+{"time":"2024-10-30T23:18:35.259239637-04:00","level":"INFO","msg":"stream: started","id":"o1t74f3e"}
+{"time":"2024-10-30T23:18:35.259262897-04:00","level":"INFO","msg":"sender: started","stream_id":"o1t74f3e"}
+{"time":"2024-10-30T23:18:35.259258987-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"o1t74f3e"}}
+{"time":"2024-10-30T23:18:35.259247477-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"o1t74f3e"}}
+{"time":"2024-10-30T23:18:35.435477397-04:00","level":"INFO","msg":"Starting system monitor"}

wandb/run-20241030_231835-o1t74f3e/logs/debug.log

@@ -0,0 +1,26 @@
+2024-10-30 23:18:35,149 INFO    MainThread:457831 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-10-30 23:18:35,149 INFO    MainThread:457831 [wandb_setup.py:_flush():79] Configure stats pid to 457831
+2024-10-30 23:18:35,149 INFO    MainThread:457831 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-10-30 23:18:35,149 INFO    MainThread:457831 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-10-30 23:18:35,149 INFO    MainThread:457831 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-30 23:18:35,149 INFO    MainThread:457831 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-30 23:18:35,149 INFO    MainThread:457831 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-10-30 23:18:35,149 INFO    MainThread:457831 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-30 23:18:35,149 INFO    MainThread:457831 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_231835-o1t74f3e/logs/debug.log
+2024-10-30 23:18:35,150 INFO    MainThread:457831 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_231835-o1t74f3e/logs/debug-internal.log
+2024-10-30 23:18:35,150 INFO    MainThread:457831 [wandb_init.py:init():621] calling init triggers
+2024-10-30 23:18:35,150 INFO    MainThread:457831 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-10-30 23:18:35,150 INFO    MainThread:457831 [wandb_init.py:init():671] starting backend
+2024-10-30 23:18:35,150 INFO    MainThread:457831 [wandb_init.py:init():675] sending inform_init request
+2024-10-30 23:18:35,151 INFO    MainThread:457831 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-30 23:18:35,151 INFO    MainThread:457831 [wandb_init.py:init():688] backend started and connected
+2024-10-30 23:18:35,154 INFO    MainThread:457831 [wandb_init.py:init():783] updated telemetry
+2024-10-30 23:18:35,179 INFO    MainThread:457831 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-10-30 23:18:35,432 INFO    MainThread:457831 [wandb_init.py:init():867] starting run threads in backend
+2024-10-30 23:18:35,527 INFO    MainThread:457831 [wandb_run.py:_console_start():2463] atexit reg
+2024-10-30 23:18:35,527 INFO    MainThread:457831 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-10-30 23:18:35,527 INFO    MainThread:457831 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-10-30 23:18:35,527 INFO    MainThread:457831 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-10-30 23:18:35,528 INFO    MainThread:457831 [wandb_init.py:init():911] run started, returning control to user process
+2024-10-30 23:18:35,529 INFO    MainThread:457831 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_full', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0}

wandb/run-20241030_233740-98qje3cr/files/output.log

@@ -0,0 +1,13 @@
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:05<00:00,  2.98s/it]
+tokenized_valid: Dataset({
+    features: ['input_ids', 'attention_mask'],
+    num_rows: 600
+})
+/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
+  warnings.warn(
+[2024-10-30 23:37:50,812] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2024-10-30 23:37:58,524] [INFO] [comm.py:652:init_distributed] cdb=None
+Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination
+Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...
+Loading extension module cpu_adam...
+Time to load cpu_adam op: 4.6087868213653564 seconds

wandb/run-20241030_233740-98qje3cr/files/requirements.txt

@@ -0,0 +1,147 @@
+funcsigs==1.0.2
+sentry-sdk==2.17.0
+multiprocess==0.70.16
+numpy==1.26.2
+pluralizer==1.2.0
+debugpy==1.6.7
+nvidia-cudnn-cu11==8.5.0.96
+deepspeed==0.15.2
+data==0.4
+pandas==2.1.3
+tomli==2.0.1
+charset-normalizer==3.3.2
+attrs==24.2.0
+aiosignal==1.3.1
+fsspec==2023.10.0
+nvidia-cusparse-cu11==11.7.4.91
+zipp==3.12.0
+mypy-extensions==1.0.0
+datasets==3.0.1
+joblib==1.3.2
+hjson==3.1.0
+traitlets==5.7.1
+stack-data==0.6.0
+transformers==4.45.1
+sympy==1.11.1
+Pygments==2.15.0
+docker-pycreds==0.4.0
+dill==0.3.8
+wheel==0.44.0
+prompt-toolkit==3.0.30
+parso==0.8.3
+ipykernel==6.23.1
+pyarrow==17.0.0
+certifi==2023.11.17
+nvidia-cufft-cu11==10.9.0.58
+six==1.16.0
+pydantic==2.9.2
+click==8.1.7
+nest-asyncio==1.5.6
+gmpy2==2.1.0
+matplotlib==3.8.2
+scipy==1.11.4
+typing_extensions==4.12.2
+statsmodels==0.14.0
+huggingface-hub==0.25.0
+frozenlist==1.4.1
+gpustat==1.1.1
+nvidia-nvtx-cu11==11.7.91
+safetensors==0.4.5
+stanza==1.9.2
+decorator==5.1.1
+seaborn==0.13.0
+sentencepiece==0.2.0
+PyYAML==6.0.1
+black==24.8.0
+protobuf==4.25.1
+pickleshare==0.7.5
+peft==0.13.0
+triton==2.0.0
+nvidia-cuda-runtime-cu11==11.7.99
+Jinja2==3.1.2
+nvidia-cusolver-cu11==11.4.0.1
+executing==1.2.0
+jupyter_client==8.1.0
+pluggy==1.3.0
+cmake==3.30.3
+pytz==2023.3.post1
+aiohappyeyeballs==2.4.2
+kiwisolver==1.4.5
+py-cpuinfo==9.0.0
+Pillow==10.1.0
+ptyprocess==0.7.0
+importlib_resources==6.4.5
+GitPython==3.1.43
+importlib-metadata==6.0.0
+iniconfig==2.0.0
+scikit-learn==1.3.2
+exceptiongroup==1.1.0
+networkx==2.8.6
+accelerate==1.0.0
+nltk==3.8.1
+shutilwhich==1.1.0
+fonttools==4.45.1
+future==0.18.3
+aiohttp==3.10.6
+wcwidth==0.2.5
+idna==3.6
+filelock==3.12.2
+pathspec==0.12.1
+jupyter_core==5.1.0
+lit==18.1.8
+nvidia-curand-cu11==10.2.10.91
+nvidia-cublas-cu11==11.10.3.66
+nvidia-ml-py==12.560.30
+msgpack==1.1.0
+python-dateutil==2.8.2
+blessed==1.20.0
+packaging==23.0
+gitdb==4.0.11
+yarl==1.13.0
+emoji==2.8.0
+tzdata==2023.3
+cycler==0.12.1
+tornado==6.2
+backcall==0.2.0
+plotnine==0.12.4
+ninja==1.11.1.1
+latex==0.7.0
+wandb==0.18.5
+setproctitle==1.3.3
+threadpoolctl==3.2.0
+requests==2.32.3
+pyparsing==3.1.1
+smmap==5.0.1
+pyzmq==23.0.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+matplotlib-inline==0.1.6
+latexcodec==1.0.0
+ipython==8.0.0
+patsy==0.5.3
+contourpy==1.2.0
+multidict==6.1.0
+mizani==0.9.3
+urllib3==2.1.0
+tokenizers==0.20.0
+MarkupSafe==2.1.2
+pip==24.2
+pexpect==4.8.0
+tqdm==4.66.5
+jedi==0.18.2
+pydantic_core==2.23.4
+tempdir==0.7.1
+mpmath==1.2.1
+setuptools==72.1.0
+pytest==7.4.3
+pure-eval==0.2.2
+psutil==5.9.1
+comm==0.1.2
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+regex==2023.10.3
+platformdirs==2.5.2
+asttokens==2.2.1
+torch==2.0.0
+nvidia-nccl-cu11==2.14.3
+xxhash==3.5.0

wandb/run-20241030_233740-98qje3cr/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-10-31T03:37:40.841144Z",
+  "args":  [
+    "--perturbation",
+    "reverse_control",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "3",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1711065919488"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241030_233740-98qje3cr/logs/debug-internal.log

@@ -0,0 +1,8 @@
+{"time":"2024-10-30T23:37:40.845317238-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-10-30T23:37:40.845328408-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_233740-98qje3cr/logs/debug-core.log"}
+{"time":"2024-10-30T23:37:40.951923871-04:00","level":"INFO","msg":"created new stream","id":"98qje3cr"}
+{"time":"2024-10-30T23:37:40.951953631-04:00","level":"INFO","msg":"stream: started","id":"98qje3cr"}
+{"time":"2024-10-30T23:37:40.952025931-04:00","level":"INFO","msg":"sender: started","stream_id":"98qje3cr"}
+{"time":"2024-10-30T23:37:40.951990871-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"98qje3cr"}}
+{"time":"2024-10-30T23:37:40.952005801-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"98qje3cr"}}
+{"time":"2024-10-30T23:37:41.405269516-04:00","level":"INFO","msg":"Starting system monitor"}

wandb/run-20241030_233740-98qje3cr/logs/debug.log

@@ -0,0 +1,26 @@
+2024-10-30 23:37:40,839 INFO    MainThread:464534 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-10-30 23:37:40,839 INFO    MainThread:464534 [wandb_setup.py:_flush():79] Configure stats pid to 464534
+2024-10-30 23:37:40,839 INFO    MainThread:464534 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-10-30 23:37:40,839 INFO    MainThread:464534 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-10-30 23:37:40,839 INFO    MainThread:464534 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-30 23:37:40,839 INFO    MainThread:464534 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-30 23:37:40,839 INFO    MainThread:464534 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-10-30 23:37:40,839 INFO    MainThread:464534 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-30 23:37:40,839 INFO    MainThread:464534 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_233740-98qje3cr/logs/debug.log
+2024-10-30 23:37:40,839 INFO    MainThread:464534 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_233740-98qje3cr/logs/debug-internal.log
+2024-10-30 23:37:40,839 INFO    MainThread:464534 [wandb_init.py:init():621] calling init triggers
+2024-10-30 23:37:40,839 INFO    MainThread:464534 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-10-30 23:37:40,839 INFO    MainThread:464534 [wandb_init.py:init():671] starting backend
+2024-10-30 23:37:40,839 INFO    MainThread:464534 [wandb_init.py:init():675] sending inform_init request
+2024-10-30 23:37:40,840 INFO    MainThread:464534 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-30 23:37:40,840 INFO    MainThread:464534 [wandb_init.py:init():688] backend started and connected
+2024-10-30 23:37:40,844 INFO    MainThread:464534 [wandb_init.py:init():783] updated telemetry
+2024-10-30 23:37:40,871 INFO    MainThread:464534 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-10-30 23:37:41,402 INFO    MainThread:464534 [wandb_init.py:init():867] starting run threads in backend
+2024-10-30 23:37:42,837 INFO    MainThread:464534 [wandb_run.py:_console_start():2463] atexit reg
+2024-10-30 23:37:42,837 INFO    MainThread:464534 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-10-30 23:37:42,837 INFO    MainThread:464534 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-10-30 23:37:42,837 INFO    MainThread:464534 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-10-30 23:37:42,860 INFO    MainThread:464534 [wandb_init.py:init():911] run started, returning control to user process
+2024-10-30 23:37:42,861 INFO    MainThread:464534 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_control', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0}

wandb/run-20241101_093116-n3c1zje4/files/output.log

@@ -0,0 +1,13 @@
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:18<00:00,  9.08s/it]
+tokenized_valid: Dataset({
+    features: ['input_ids', 'attention_mask'],
+    num_rows: 600
+})
+/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
+  warnings.warn(
+[2024-11-01 09:31:37,042] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2024-11-01 09:31:45,660] [INFO] [comm.py:652:init_distributed] cdb=None
+Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination
+Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...
+Loading extension module cpu_adam...
+Time to load cpu_adam op: 4.823585748672485 seconds

wandb/run-20241101_093116-n3c1zje4/files/requirements.txt

@@ -0,0 +1,147 @@
+funcsigs==1.0.2
+sentry-sdk==2.17.0
+multiprocess==0.70.16
+numpy==1.26.2
+pluralizer==1.2.0
+debugpy==1.6.7
+nvidia-cudnn-cu11==8.5.0.96
+deepspeed==0.15.2
+data==0.4
+pandas==2.1.3
+tomli==2.0.1
+charset-normalizer==3.3.2
+attrs==24.2.0
+aiosignal==1.3.1
+fsspec==2023.10.0
+nvidia-cusparse-cu11==11.7.4.91
+zipp==3.12.0
+mypy-extensions==1.0.0
+datasets==3.0.1
+joblib==1.3.2
+hjson==3.1.0
+traitlets==5.7.1
+stack-data==0.6.0
+transformers==4.45.1
+sympy==1.11.1
+Pygments==2.15.0
+docker-pycreds==0.4.0
+dill==0.3.8
+wheel==0.44.0
+prompt-toolkit==3.0.30
+parso==0.8.3
+ipykernel==6.23.1
+pyarrow==17.0.0
+certifi==2023.11.17
+nvidia-cufft-cu11==10.9.0.58
+six==1.16.0
+pydantic==2.9.2
+click==8.1.7
+nest-asyncio==1.5.6
+gmpy2==2.1.0
+matplotlib==3.8.2
+scipy==1.11.4
+typing_extensions==4.12.2
+statsmodels==0.14.0
+huggingface-hub==0.25.0
+frozenlist==1.4.1
+gpustat==1.1.1
+nvidia-nvtx-cu11==11.7.91
+safetensors==0.4.5
+stanza==1.9.2
+decorator==5.1.1
+seaborn==0.13.0
+sentencepiece==0.2.0
+PyYAML==6.0.1
+black==24.8.0
+protobuf==4.25.1
+pickleshare==0.7.5
+peft==0.13.0
+triton==2.0.0
+nvidia-cuda-runtime-cu11==11.7.99
+Jinja2==3.1.2
+nvidia-cusolver-cu11==11.4.0.1
+executing==1.2.0
+jupyter_client==8.1.0
+pluggy==1.3.0
+cmake==3.30.3
+pytz==2023.3.post1
+aiohappyeyeballs==2.4.2
+kiwisolver==1.4.5
+py-cpuinfo==9.0.0
+Pillow==10.1.0
+ptyprocess==0.7.0
+importlib_resources==6.4.5
+GitPython==3.1.43
+importlib-metadata==6.0.0
+iniconfig==2.0.0
+scikit-learn==1.3.2
+exceptiongroup==1.1.0
+networkx==2.8.6
+accelerate==1.0.0
+nltk==3.8.1
+shutilwhich==1.1.0
+fonttools==4.45.1
+future==0.18.3
+aiohttp==3.10.6
+wcwidth==0.2.5
+idna==3.6
+filelock==3.12.2
+pathspec==0.12.1
+jupyter_core==5.1.0
+lit==18.1.8
+nvidia-curand-cu11==10.2.10.91
+nvidia-cublas-cu11==11.10.3.66
+nvidia-ml-py==12.560.30
+msgpack==1.1.0
+python-dateutil==2.8.2
+blessed==1.20.0
+packaging==23.0
+gitdb==4.0.11
+yarl==1.13.0
+emoji==2.8.0
+tzdata==2023.3
+cycler==0.12.1
+tornado==6.2
+backcall==0.2.0
+plotnine==0.12.4
+ninja==1.11.1.1
+latex==0.7.0
+wandb==0.18.5
+setproctitle==1.3.3
+threadpoolctl==3.2.0
+requests==2.32.3
+pyparsing==3.1.1
+smmap==5.0.1
+pyzmq==23.0.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+matplotlib-inline==0.1.6
+latexcodec==1.0.0
+ipython==8.0.0
+patsy==0.5.3
+contourpy==1.2.0
+multidict==6.1.0
+mizani==0.9.3
+urllib3==2.1.0
+tokenizers==0.20.0
+MarkupSafe==2.1.2
+pip==24.2
+pexpect==4.8.0
+tqdm==4.66.5
+jedi==0.18.2
+pydantic_core==2.23.4
+tempdir==0.7.1
+mpmath==1.2.1
+setuptools==72.1.0
+pytest==7.4.3
+pure-eval==0.2.2
+psutil==5.9.1
+comm==0.1.2
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+regex==2023.10.3
+platformdirs==2.5.2
+asttokens==2.2.1
+torch==2.0.0
+nvidia-nccl-cu11==2.14.3
+xxhash==3.5.0

wandb/run-20241101_093116-n3c1zje4/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-11-01T13:31:16.510021Z",
+  "args":  [
+    "--perturbation",
+    "reverse_control",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "7",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1754716262400"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241101_093116-n3c1zje4/logs/debug-internal.log

@@ -0,0 +1,8 @@
+{"time":"2024-11-01T09:31:16.512866181-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-11-01T09:31:16.512881061-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_093116-n3c1zje4/logs/debug-core.log"}
+{"time":"2024-11-01T09:31:16.623420531-04:00","level":"INFO","msg":"created new stream","id":"n3c1zje4"}
+{"time":"2024-11-01T09:31:16.623477982-04:00","level":"INFO","msg":"stream: started","id":"n3c1zje4"}
+{"time":"2024-11-01T09:31:16.623527602-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"n3c1zje4"}}
+{"time":"2024-11-01T09:31:16.623492722-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"n3c1zje4"}}
+{"time":"2024-11-01T09:31:16.623569862-04:00","level":"INFO","msg":"sender: started","stream_id":"n3c1zje4"}
+{"time":"2024-11-01T09:31:16.840771598-04:00","level":"INFO","msg":"Starting system monitor"}

wandb/run-20241101_093116-n3c1zje4/logs/debug.log

@@ -0,0 +1,26 @@
+2024-11-01 09:31:16,508 INFO    MainThread:781950 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-11-01 09:31:16,508 INFO    MainThread:781950 [wandb_setup.py:_flush():79] Configure stats pid to 781950
+2024-11-01 09:31:16,508 INFO    MainThread:781950 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-11-01 09:31:16,508 INFO    MainThread:781950 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-11-01 09:31:16,508 INFO    MainThread:781950 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-11-01 09:31:16,508 INFO    MainThread:781950 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-11-01 09:31:16,508 INFO    MainThread:781950 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-11-01 09:31:16,508 INFO    MainThread:781950 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-11-01 09:31:16,508 INFO    MainThread:781950 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_093116-n3c1zje4/logs/debug.log
+2024-11-01 09:31:16,508 INFO    MainThread:781950 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_093116-n3c1zje4/logs/debug-internal.log
+2024-11-01 09:31:16,508 INFO    MainThread:781950 [wandb_init.py:init():621] calling init triggers
+2024-11-01 09:31:16,508 INFO    MainThread:781950 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-11-01 09:31:16,508 INFO    MainThread:781950 [wandb_init.py:init():671] starting backend
+2024-11-01 09:31:16,508 INFO    MainThread:781950 [wandb_init.py:init():675] sending inform_init request
+2024-11-01 09:31:16,509 INFO    MainThread:781950 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-11-01 09:31:16,509 INFO    MainThread:781950 [wandb_init.py:init():688] backend started and connected
+2024-11-01 09:31:16,513 INFO    MainThread:781950 [wandb_init.py:init():783] updated telemetry
+2024-11-01 09:31:16,546 INFO    MainThread:781950 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-11-01 09:31:16,836 INFO    MainThread:781950 [wandb_init.py:init():867] starting run threads in backend
+2024-11-01 09:31:16,962 INFO    MainThread:781950 [wandb_run.py:_console_start():2463] atexit reg
+2024-11-01 09:31:16,962 INFO    MainThread:781950 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-11-01 09:31:16,963 INFO    MainThread:781950 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-11-01 09:31:16,963 INFO    MainThread:781950 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-11-01 09:31:16,964 INFO    MainThread:781950 [wandb_init.py:init():911] run started, returning control to user process
+2024-11-01 09:31:16,964 INFO    MainThread:781950 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_control', 'train_set': '10M', 'batch_size': 3, 'epoch': 7, 'seed': 0, 'lr': 5e-06}

wandb/run-20241101_200535-kh416n4m/run-kh416n4m.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:262a348816d9419ec3403966b5e76fe2cb0a701d902903bc04dc131eabf5c8f4
+size 131072

wandb/run-20241101_202058-jijqbvs1/run-jijqbvs1.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6daebef05a7995ff2c2936db987cd658c3b9b8ee650c82f84f0b76c724dcff4
+size 13287862

wandb/run-20241105_160652-il3id7gi/files/config.yaml

@@ -0,0 +1,49 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.9.19
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "3":
+                - 13
+                - 23
+                - 55
+            "4": 3.9.19
+            "5": 0.18.5
+            "6": 4.45.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+batch_size:
+    value: 3
+epoch:
+    value: 3
+lr:
+    value: 5e-06
+perturbation:
+    value: shuffle_deterministic21
+seed:
+    value: 0
+train_set:
+    value: 10M

wandb/run-20241105_160652-il3id7gi/files/output.log

@@ -0,0 +1,8 @@
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 165, in <module>
+    dataset = load_dataset('babylm_dataset_test.py', name=dataset_name, trust_remote_code=True)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/load.py", line 2096, in load_dataset
+    builder_instance.download_and_prepare(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/builder.py", line 875, in download_and_prepare
+    raise OSError(
+OSError: Not enough disk space. Needed: Unknown size (download: Unknown size, generated: Unknown size, post-processed: Unknown size)

wandb/run-20241105_160652-il3id7gi/files/requirements.txt

@@ -0,0 +1,147 @@
+funcsigs==1.0.2
+sentry-sdk==2.17.0
+multiprocess==0.70.16
+numpy==1.26.2
+pluralizer==1.2.0
+debugpy==1.6.7
+nvidia-cudnn-cu11==8.5.0.96
+deepspeed==0.15.2
+data==0.4
+pandas==2.1.3
+tomli==2.0.1
+charset-normalizer==3.3.2
+attrs==24.2.0
+aiosignal==1.3.1
+fsspec==2023.10.0
+nvidia-cusparse-cu11==11.7.4.91
+zipp==3.12.0
+mypy-extensions==1.0.0
+datasets==3.0.1
+joblib==1.3.2
+hjson==3.1.0
+traitlets==5.7.1
+stack-data==0.6.0
+transformers==4.45.1
+sympy==1.11.1
+Pygments==2.15.0
+docker-pycreds==0.4.0
+dill==0.3.8
+wheel==0.44.0
+prompt-toolkit==3.0.30
+parso==0.8.3
+ipykernel==6.23.1
+pyarrow==17.0.0
+certifi==2023.11.17
+nvidia-cufft-cu11==10.9.0.58
+six==1.16.0
+pydantic==2.9.2
+click==8.1.7
+nest-asyncio==1.5.6
+gmpy2==2.1.0
+matplotlib==3.8.2
+scipy==1.11.4
+typing_extensions==4.12.2
+statsmodels==0.14.0
+huggingface-hub==0.25.0
+frozenlist==1.4.1
+gpustat==1.1.1
+nvidia-nvtx-cu11==11.7.91
+safetensors==0.4.5
+stanza==1.9.2
+decorator==5.1.1
+seaborn==0.13.0
+sentencepiece==0.2.0
+PyYAML==6.0.1
+black==24.8.0
+protobuf==4.25.1
+pickleshare==0.7.5
+peft==0.13.0
+triton==2.0.0
+nvidia-cuda-runtime-cu11==11.7.99
+Jinja2==3.1.2
+nvidia-cusolver-cu11==11.4.0.1
+executing==1.2.0
+jupyter_client==8.1.0
+pluggy==1.3.0
+cmake==3.30.3
+pytz==2023.3.post1
+aiohappyeyeballs==2.4.2
+kiwisolver==1.4.5
+py-cpuinfo==9.0.0
+Pillow==10.1.0
+ptyprocess==0.7.0
+importlib_resources==6.4.5
+GitPython==3.1.43
+importlib-metadata==6.0.0
+iniconfig==2.0.0
+scikit-learn==1.3.2
+exceptiongroup==1.1.0
+networkx==2.8.6
+accelerate==1.0.0
+nltk==3.8.1
+shutilwhich==1.1.0
+fonttools==4.45.1
+future==0.18.3
+aiohttp==3.10.6
+wcwidth==0.2.5
+idna==3.6
+filelock==3.12.2
+pathspec==0.12.1
+jupyter_core==5.1.0
+lit==18.1.8
+nvidia-curand-cu11==10.2.10.91
+nvidia-cublas-cu11==11.10.3.66
+nvidia-ml-py==12.560.30
+msgpack==1.1.0
+python-dateutil==2.8.2
+blessed==1.20.0
+packaging==23.0
+gitdb==4.0.11
+yarl==1.13.0
+emoji==2.8.0
+tzdata==2023.3
+cycler==0.12.1
+tornado==6.2
+backcall==0.2.0
+plotnine==0.12.4
+ninja==1.11.1.1
+latex==0.7.0
+wandb==0.18.5
+setproctitle==1.3.3
+threadpoolctl==3.2.0
+requests==2.32.3
+pyparsing==3.1.1
+smmap==5.0.1
+pyzmq==23.0.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+matplotlib-inline==0.1.6
+latexcodec==1.0.0
+ipython==8.0.0
+patsy==0.5.3
+contourpy==1.2.0
+multidict==6.1.0
+mizani==0.9.3
+urllib3==2.1.0
+tokenizers==0.20.0
+MarkupSafe==2.1.2
+pip==24.2
+pexpect==4.8.0
+tqdm==4.66.5
+jedi==0.18.2
+pydantic_core==2.23.4
+tempdir==0.7.1
+mpmath==1.2.1
+setuptools==72.1.0
+pytest==7.4.3
+pure-eval==0.2.2
+psutil==5.9.1
+comm==0.1.2
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+regex==2023.10.3
+platformdirs==2.5.2
+asttokens==2.2.1
+torch==2.0.0
+nvidia-nccl-cu11==2.14.3
+xxhash==3.5.0

wandb/run-20241105_160652-il3id7gi/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-11-05T21:06:52.164993Z",
+  "args":  [
+    "--perturbation",
+    "shuffle_deterministic21",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "3",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1792542826496"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241105_160652-il3id7gi/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":5}}

wandb/run-20241105_160652-il3id7gi/logs/debug-internal.log

@@ -0,0 +1,17 @@
+{"time":"2024-11-05T16:06:52.167714581-05:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-11-05T16:06:52.167734171-05:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_160652-il3id7gi/logs/debug-core.log"}
+{"time":"2024-11-05T16:06:52.276915561-05:00","level":"INFO","msg":"created new stream","id":"il3id7gi"}
+{"time":"2024-11-05T16:06:52.276986211-05:00","level":"INFO","msg":"stream: started","id":"il3id7gi"}
+{"time":"2024-11-05T16:06:52.277107652-05:00","level":"INFO","msg":"sender: started","stream_id":"il3id7gi"}
+{"time":"2024-11-05T16:06:52.277018542-05:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"il3id7gi"}}
+{"time":"2024-11-05T16:06:52.277107022-05:00","level":"INFO","msg":"handler: started","stream_id":{"value":"il3id7gi"}}
+{"time":"2024-11-05T16:06:52.50033616-05:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-11-05T16:06:57.72841623-05:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-11-05T16:06:57.72836442-05:00","level":"INFO","msg":"stream: closing","id":"il3id7gi"}
+{"time":"2024-11-05T16:06:57.729503476-05:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-11-05T16:06:57.814719044-05:00","level":"ERROR","msg":"sender: sendDefer: failed to build job artifact","error":"failed to write data to file: write /tmp/tmpfile-193938247: no space left on device"}
+{"time":"2024-11-05T16:06:58.075824688-05:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-11-05T16:06:58.203385044-05:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"il3id7gi"}}
+{"time":"2024-11-05T16:06:58.203426045-05:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"il3id7gi"}}
+{"time":"2024-11-05T16:06:58.203465995-05:00","level":"INFO","msg":"sender: closed","stream_id":"il3id7gi"}
+{"time":"2024-11-05T16:06:58.203502235-05:00","level":"INFO","msg":"stream: closed","id":"il3id7gi"}

wandb/run-20241105_160652-il3id7gi/logs/debug.log

@@ -0,0 +1,27 @@
+2024-11-05 16:06:52,161 INFO    MainThread:1771275 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-11-05 16:06:52,161 INFO    MainThread:1771275 [wandb_setup.py:_flush():79] Configure stats pid to 1771275
+2024-11-05 16:06:52,161 INFO    MainThread:1771275 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-11-05 16:06:52,161 INFO    MainThread:1771275 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-11-05 16:06:52,161 INFO    MainThread:1771275 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-11-05 16:06:52,161 INFO    MainThread:1771275 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-11-05 16:06:52,161 INFO    MainThread:1771275 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-11-05 16:06:52,161 INFO    MainThread:1771275 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-11-05 16:06:52,161 INFO    MainThread:1771275 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_160652-il3id7gi/logs/debug.log
+2024-11-05 16:06:52,161 INFO    MainThread:1771275 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_160652-il3id7gi/logs/debug-internal.log
+2024-11-05 16:06:52,162 INFO    MainThread:1771275 [wandb_init.py:init():621] calling init triggers
+2024-11-05 16:06:52,162 INFO    MainThread:1771275 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-11-05 16:06:52,162 INFO    MainThread:1771275 [wandb_init.py:init():671] starting backend
+2024-11-05 16:06:52,162 INFO    MainThread:1771275 [wandb_init.py:init():675] sending inform_init request
+2024-11-05 16:06:52,164 INFO    MainThread:1771275 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-11-05 16:06:52,164 INFO    MainThread:1771275 [wandb_init.py:init():688] backend started and connected
+2024-11-05 16:06:52,167 INFO    MainThread:1771275 [wandb_init.py:init():783] updated telemetry
+2024-11-05 16:06:52,198 INFO    MainThread:1771275 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-11-05 16:06:52,497 INFO    MainThread:1771275 [wandb_init.py:init():867] starting run threads in backend
+2024-11-05 16:06:52,584 INFO    MainThread:1771275 [wandb_run.py:_console_start():2463] atexit reg
+2024-11-05 16:06:52,584 INFO    MainThread:1771275 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-11-05 16:06:52,584 INFO    MainThread:1771275 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-11-05 16:06:52,584 INFO    MainThread:1771275 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-11-05 16:06:52,586 INFO    MainThread:1771275 [wandb_init.py:init():911] run started, returning control to user process
+2024-11-05 16:06:52,586 INFO    MainThread:1771275 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_deterministic21', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0, 'lr': 5e-06}
+2024-11-05 16:06:57,728 WARNING MsgRouterThr:1771275 [router.py:message_loop():77] message_loop has been closed

wandb/run-20241105_163039-q4e8d8hm/files/config.yaml

@@ -0,0 +1,49 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.9.19
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "3":
+                - 13
+                - 23
+                - 55
+            "4": 3.9.19
+            "5": 0.18.5
+            "6": 4.45.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+batch_size:
+    value: 3
+epoch:
+    value: 3
+lr:
+    value: 5e-06
+perturbation:
+    value: shuffle_deterministic21
+seed:
+    value: 0
+train_set:
+    value: 10M

wandb/run-20241105_163039-q4e8d8hm/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-11-05T21:30:39.177983Z",
+  "args":  [
+    "--perturbation",
+    "shuffle_deterministic21",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "3",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1785984999424"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241105_163039-q4e8d8hm/logs/debug.log

@@ -0,0 +1,27 @@
+2024-11-05 16:30:39,175 INFO    MainThread:1780128 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-11-05 16:30:39,175 INFO    MainThread:1780128 [wandb_setup.py:_flush():79] Configure stats pid to 1780128
+2024-11-05 16:30:39,175 INFO    MainThread:1780128 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-11-05 16:30:39,175 INFO    MainThread:1780128 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-11-05 16:30:39,175 INFO    MainThread:1780128 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-11-05 16:30:39,175 INFO    MainThread:1780128 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-11-05 16:30:39,175 INFO    MainThread:1780128 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-11-05 16:30:39,175 INFO    MainThread:1780128 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-11-05 16:30:39,175 INFO    MainThread:1780128 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_163039-q4e8d8hm/logs/debug.log
+2024-11-05 16:30:39,175 INFO    MainThread:1780128 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_163039-q4e8d8hm/logs/debug-internal.log
+2024-11-05 16:30:39,175 INFO    MainThread:1780128 [wandb_init.py:init():621] calling init triggers
+2024-11-05 16:30:39,175 INFO    MainThread:1780128 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-11-05 16:30:39,175 INFO    MainThread:1780128 [wandb_init.py:init():671] starting backend
+2024-11-05 16:30:39,176 INFO    MainThread:1780128 [wandb_init.py:init():675] sending inform_init request
+2024-11-05 16:30:39,177 INFO    MainThread:1780128 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-11-05 16:30:39,177 INFO    MainThread:1780128 [wandb_init.py:init():688] backend started and connected
+2024-11-05 16:30:39,181 INFO    MainThread:1780128 [wandb_init.py:init():783] updated telemetry
+2024-11-05 16:30:39,205 INFO    MainThread:1780128 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-11-05 16:30:40,353 INFO    MainThread:1780128 [wandb_init.py:init():867] starting run threads in backend
+2024-11-05 16:30:40,466 INFO    MainThread:1780128 [wandb_run.py:_console_start():2463] atexit reg
+2024-11-05 16:30:40,466 INFO    MainThread:1780128 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-11-05 16:30:40,466 INFO    MainThread:1780128 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-11-05 16:30:40,466 INFO    MainThread:1780128 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-11-05 16:30:40,467 INFO    MainThread:1780128 [wandb_init.py:init():911] run started, returning control to user process
+2024-11-05 16:30:40,468 INFO    MainThread:1780128 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_deterministic21', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0, 'lr': 5e-06}
+2024-11-05 16:32:00,266 WARNING MsgRouterThr:1780128 [router.py:message_loop():77] message_loop has been closed

wandb/run-20241105_163244-o1vw2gev/logs/debug-internal.log

@@ -0,0 +1,8 @@
+{"time":"2024-11-05T16:32:44.915617704-05:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-11-05T16:32:44.915629374-05:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_163244-o1vw2gev/logs/debug-core.log"}
+{"time":"2024-11-05T16:32:45.021934355-05:00","level":"INFO","msg":"created new stream","id":"o1vw2gev"}
+{"time":"2024-11-05T16:32:45.021967635-05:00","level":"INFO","msg":"stream: started","id":"o1vw2gev"}
+{"time":"2024-11-05T16:32:45.022053626-05:00","level":"INFO","msg":"sender: started","stream_id":"o1vw2gev"}
+{"time":"2024-11-05T16:32:45.022012265-05:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"o1vw2gev"}}
+{"time":"2024-11-05T16:32:45.022137556-05:00","level":"INFO","msg":"handler: started","stream_id":{"value":"o1vw2gev"}}
+{"time":"2024-11-05T16:32:45.194150228-05:00","level":"INFO","msg":"Starting system monitor"}

wandb/run-20241105_223736-2kagvv0a/files/config.yaml

@@ -0,0 +1,50 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.9.19
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "3":
+                - 2
+                - 13
+                - 23
+                - 55
+            "4": 3.9.19
+            "5": 0.18.5
+            "6": 4.45.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+batch_size:
+    value: 3
+epoch:
+    value: 3
+lr:
+    value: 5e-06
+perturbation:
+    value: shuffle_deterministic21
+seed:
+    value: 0
+train_set:
+    value: 10M

wandb/run-20241105_223736-2kagvv0a/files/output.log

@@ -0,0 +1,18 @@
+wandb: 500 encountered ({"errors":[{"message":"An internal error occurred. Please contact support.","path":["upsertBucket"]}],"data":{"upsertBucket":null}}), retrying request
+Downloading shards: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [02:32<00:00, 76.10s/it]
+Loading checkpoint shards: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:04<00:00,  2.45s/it]
+tokenized_valid: Dataset({
+    features: ['input_ids', 'attention_mask'],
+    num_rows: 1000
+})
+/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
+  warnings.warn(
+[2024-11-05 22:40:19,363] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2024-11-05 22:40:27,697] [INFO] [comm.py:652:init_distributed] cdb=None
+Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination
+Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...
+Emitting ninja build file /home/chunhui/.cache/torch_extensions/py39_cu117/cpu_adam/build.ninja...
+Building extension module cpu_adam...
+Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N)
+Loading extension module cpu_adam...
+Time to load cpu_adam op: 5.439234256744385 seconds

wandb/run-20241105_223736-2kagvv0a/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-11-06T03:37:36.194798Z",
+  "args":  [
+    "--perturbation",
+    "shuffle_deterministic21",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "3",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1785078390784"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241105_223736-2kagvv0a/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":77797}}

wandb/run-20241105_223736-2kagvv0a/logs/debug-internal.log

@@ -0,0 +1,18 @@
+{"time":"2024-11-05T22:37:36.202357237-05:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-11-05T22:37:36.202369547-05:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_223736-2kagvv0a/logs/debug-core.log"}
+{"time":"2024-11-05T22:37:36.311435374-05:00","level":"INFO","msg":"created new stream","id":"2kagvv0a"}
+{"time":"2024-11-05T22:37:36.311490154-05:00","level":"INFO","msg":"stream: started","id":"2kagvv0a"}
+{"time":"2024-11-05T22:37:36.311518374-05:00","level":"INFO","msg":"sender: started","stream_id":"2kagvv0a"}
+{"time":"2024-11-05T22:37:36.311506174-05:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"2kagvv0a"}}
+{"time":"2024-11-05T22:37:36.311515264-05:00","level":"INFO","msg":"handler: started","stream_id":{"value":"2kagvv0a"}}
+{"time":"2024-11-05T22:37:36.430260321-05:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-11-05T22:37:38.657390136-05:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-11-06T20:14:13.275515344-05:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-11-06T20:14:13.36533327-05:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-11-06T20:14:14.247035454-05:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"saving job artifact","runtime_seconds":0.766071365,"subtasks":[{"desc":"requirements.frozen.txt","runtime_seconds":0.267306342,"progress":"2.6KB/2.6KB"}]}],"total_operations":1}}
+{"time":"2024-11-06T20:14:14.897915803-05:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-11-06T20:14:16.038235461-05:00","level":"INFO","msg":"stream: closing","id":"2kagvv0a"}
+{"time":"2024-11-06T20:14:16.038268531-05:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"2kagvv0a"}}
+{"time":"2024-11-06T20:14:16.038289471-05:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"2kagvv0a"}}
+{"time":"2024-11-06T20:14:16.038310401-05:00","level":"INFO","msg":"sender: closed","stream_id":"2kagvv0a"}
+{"time":"2024-11-06T20:14:16.038388792-05:00","level":"INFO","msg":"stream: closed","id":"2kagvv0a"}

wandb/run-20241105_223736-2kagvv0a/logs/debug.log

@@ -0,0 +1,33 @@
+2024-11-05 22:37:36,192 INFO    MainThread:1802405 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-11-05 22:37:36,193 INFO    MainThread:1802405 [wandb_setup.py:_flush():79] Configure stats pid to 1802405
+2024-11-05 22:37:36,193 INFO    MainThread:1802405 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-11-05 22:37:36,193 INFO    MainThread:1802405 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-11-05 22:37:36,193 INFO    MainThread:1802405 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-11-05 22:37:36,193 INFO    MainThread:1802405 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-11-05 22:37:36,193 INFO    MainThread:1802405 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-11-05 22:37:36,193 INFO    MainThread:1802405 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-11-05 22:37:36,193 INFO    MainThread:1802405 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_223736-2kagvv0a/logs/debug.log
+2024-11-05 22:37:36,193 INFO    MainThread:1802405 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_223736-2kagvv0a/logs/debug-internal.log
+2024-11-05 22:37:36,193 INFO    MainThread:1802405 [wandb_init.py:init():621] calling init triggers
+2024-11-05 22:37:36,193 INFO    MainThread:1802405 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-11-05 22:37:36,193 INFO    MainThread:1802405 [wandb_init.py:init():671] starting backend
+2024-11-05 22:37:36,193 INFO    MainThread:1802405 [wandb_init.py:init():675] sending inform_init request
+2024-11-05 22:37:36,194 INFO    MainThread:1802405 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-11-05 22:37:36,194 INFO    MainThread:1802405 [wandb_init.py:init():688] backend started and connected
+2024-11-05 22:37:36,197 INFO    MainThread:1802405 [wandb_init.py:init():783] updated telemetry
+2024-11-05 22:37:36,222 INFO    MainThread:1802405 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-11-05 22:37:38,650 INFO    MainThread:1802405 [wandb_init.py:init():867] starting run threads in backend
+2024-11-05 22:37:38,746 INFO    MainThread:1802405 [wandb_run.py:_console_start():2463] atexit reg
+2024-11-05 22:37:38,746 INFO    MainThread:1802405 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-11-05 22:37:38,746 INFO    MainThread:1802405 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-11-05 22:37:38,746 INFO    MainThread:1802405 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-11-05 22:37:38,747 INFO    MainThread:1802405 [wandb_init.py:init():911] run started, returning control to user process
+2024-11-05 22:37:38,747 INFO    MainThread:1802405 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_deterministic21', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0, 'lr': 5e-06}
+2024-11-06 20:14:13,216 INFO    MainThread:1802405 [wandb_run.py:_finish():2158] finishing run yaning1001-dartmouth-college/exp-impo-shuffle/2kagvv0a
+2024-11-06 20:14:13,227 INFO    MainThread:1802405 [wandb_run.py:_atexit_cleanup():2426] got exitcode: 0
+2024-11-06 20:14:13,227 INFO    MainThread:1802405 [wandb_run.py:_restore():2408] restore
+2024-11-06 20:14:13,246 INFO    MainThread:1802405 [wandb_run.py:_restore():2414] restore done
+2024-11-06 20:14:16,032 INFO    MainThread:1802405 [wandb_run.py:_footer_history_summary_info():3975] rendering history
+2024-11-06 20:14:16,032 INFO    MainThread:1802405 [wandb_run.py:_footer_history_summary_info():4007] rendering summary
+2024-11-06 20:14:16,037 INFO    MainThread:1802405 [wandb_run.py:_footer_sync_info():3934] logging synced files

wandb/run-20241106_203647-jauklpbe/files/config.yaml

@@ -0,0 +1,49 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.9.19
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "3":
+                - 13
+                - 23
+                - 55
+            "4": 3.9.19
+            "5": 0.18.5
+            "6": 4.45.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+batch_size:
+    value: 3
+epoch:
+    value: 3
+lr:
+    value: 5e-06
+perturbation:
+    value: shuffle_deterministic84
+seed:
+    value: 0
+train_set:
+    value: 10M

wandb/run-20241106_203647-jauklpbe/files/output.log

@@ -0,0 +1,60 @@
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/utils/_http.py", line 406, in hf_raise_for_status
+    response.raise_for_status()
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/requests/models.py", line 1024, in raise_for_status
+    raise HTTPError(http_error_msg, response=self)
+requests.exceptions.HTTPError: 401 Client Error: Unauthorized for url: https://huggingface.co/meta-llama/Llama-3.2-3B/resolve/main/config.json
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/utils/hub.py", line 403, in cached_file
+    resolved_file = hf_hub_download(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/utils/_deprecation.py", line 101, in inner_f
+    return f(*args, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn
+    return fn(*args, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1232, in hf_hub_download
+    return _hf_hub_download_to_cache_dir(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1339, in _hf_hub_download_to_cache_dir
+    _raise_on_head_call_error(head_call_error, force_download, local_files_only)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1854, in _raise_on_head_call_error
+    raise head_call_error
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1746, in _get_metadata_or_catch_error
+    metadata = get_hf_file_metadata(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn
+    return fn(*args, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1666, in get_hf_file_metadata
+    r = _request_wrapper(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 364, in _request_wrapper
+    response = _request_wrapper(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 388, in _request_wrapper
+    hf_raise_for_status(response)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/utils/_http.py", line 423, in hf_raise_for_status
+    raise _format(GatedRepoError, message, response) from e
+huggingface_hub.errors.GatedRepoError: 401 Client Error. (Request ID: Root=1-672c19c6-4ab49c2772cb92404817a4a6;d7309389-7bc2-43ca-8268-d31dc94d9f27)
+
+Cannot access gated repo for url https://huggingface.co/meta-llama/Llama-3.2-3B/resolve/main/config.json.
+Access to model meta-llama/Llama-3.2-3B is restricted. You must have access to it and be authenticated to access it. Please log in.
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 174, in <module>
+    model = AutoModelForCausalLM.from_pretrained(model_name,
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/models/auto/auto_factory.py", line 526, in from_pretrained
+    config, kwargs = AutoConfig.from_pretrained(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/models/auto/configuration_auto.py", line 1006, in from_pretrained
+    config_dict, unused_kwargs = PretrainedConfig.get_config_dict(pretrained_model_name_or_path, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/configuration_utils.py", line 567, in get_config_dict
+    config_dict, kwargs = cls._get_config_dict(pretrained_model_name_or_path, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/configuration_utils.py", line 626, in _get_config_dict
+    resolved_config_file = cached_file(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/utils/hub.py", line 421, in cached_file
+    raise EnvironmentError(
+OSError: You are trying to access a gated repo.
+Make sure to have access to it at https://huggingface.co/meta-llama/Llama-3.2-3B.
+401 Client Error. (Request ID: Root=1-672c19c6-4ab49c2772cb92404817a4a6;d7309389-7bc2-43ca-8268-d31dc94d9f27)
+
+Cannot access gated repo for url https://huggingface.co/meta-llama/Llama-3.2-3B/resolve/main/config.json.
+Access to model meta-llama/Llama-3.2-3B is restricted. You must have access to it and be authenticated to access it. Please log in.

wandb/run-20241106_203647-jauklpbe/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-11-07T01:36:47.516115Z",
+  "args":  [
+    "--perturbation",
+    "shuffle_deterministic84",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "3",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1791616151552"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241106_203647-jauklpbe/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":23}}