Add files using upload-large-folder tool

.gitattributes

@@ -92,3 +92,8 @@ wandb/run-20241106_232725-f16bcfrx/run-f16bcfrx.wandb filter=lfs diff=lfs merge=
 wandb/run-20241105_163244-59l4qxgx/run-59l4qxgx.wandb filter=lfs diff=lfs merge=lfs -text
 wandb/run-20241101_202058-ptl7coag/run-ptl7coag.wandb filter=lfs diff=lfs merge=lfs -text
 wandb/run-20241129_235322-bxqdruiw/run-bxqdruiw.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241030_233740-np98q8en/run-np98q8en.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241101_202058-hjyig8so/run-hjyig8so.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241101_012733-3tsgnm2p/run-3tsgnm2p.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241030_112852-av3r7rx8/run-av3r7rx8.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241115_125218-rrve0rbk/run-rrve0rbk.wandb filter=lfs diff=lfs merge=lfs -text

wandb/run-20241030_013339-pahr4hk1/files/output.log

@@ -0,0 +1,21 @@
+config.json: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 844/844 [00:00<00:00, 279kB/s]
+model.safetensors.index.json: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████| 20.9k/20.9k [00:00<00:00, 18.9MB/s]
+model-00001-of-00002.safetensors: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████| 4.97G/4.97G [01:57<00:00, 42.1MB/s]
+model-00002-of-00002.safetensors: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████| 1.46G/1.46G [00:34<00:00, 42.5MB/s]
+Downloading shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [02:32<00:00, 76.19s/it]
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:04<00:00,  2.10s/it]
+generation_config.json: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 185/185 [00:00<00:00, 74.7kB/s]
+Map: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 17519/17519 [00:57<00:00, 302.93 examples/s]
+Map: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 18140/18140 [00:57<00:00, 316.26 examples/s]
+tokenized_valid: Dataset({
+    features: ['input_ids', 'attention_mask'],
+    num_rows: 600
+})
+/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
+  warnings.warn(
+[2024-10-30 01:38:13,749] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2024-10-30 01:38:21,729] [INFO] [comm.py:652:init_distributed] cdb=None
+Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination
+Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...
+Loading extension module cpu_adam...
+Time to load cpu_adam op: 4.509259939193726 seconds

wandb/run-20241030_013339-pahr4hk1/files/requirements.txt

@@ -0,0 +1,147 @@
+funcsigs==1.0.2
+sentry-sdk==2.17.0
+multiprocess==0.70.16
+numpy==1.26.2
+pluralizer==1.2.0
+debugpy==1.6.7
+nvidia-cudnn-cu11==8.5.0.96
+deepspeed==0.15.2
+data==0.4
+pandas==2.1.3
+tomli==2.0.1
+charset-normalizer==3.3.2
+attrs==24.2.0
+aiosignal==1.3.1
+fsspec==2023.10.0
+nvidia-cusparse-cu11==11.7.4.91
+zipp==3.12.0
+mypy-extensions==1.0.0
+datasets==3.0.1
+joblib==1.3.2
+hjson==3.1.0
+traitlets==5.7.1
+stack-data==0.6.0
+transformers==4.45.1
+sympy==1.11.1
+Pygments==2.15.0
+docker-pycreds==0.4.0
+dill==0.3.8
+wheel==0.44.0
+prompt-toolkit==3.0.30
+parso==0.8.3
+ipykernel==6.23.1
+pyarrow==17.0.0
+certifi==2023.11.17
+nvidia-cufft-cu11==10.9.0.58
+six==1.16.0
+pydantic==2.9.2
+click==8.1.7
+nest-asyncio==1.5.6
+gmpy2==2.1.0
+matplotlib==3.8.2
+scipy==1.11.4
+typing_extensions==4.12.2
+statsmodels==0.14.0
+huggingface-hub==0.25.0
+frozenlist==1.4.1
+gpustat==1.1.1
+nvidia-nvtx-cu11==11.7.91
+safetensors==0.4.5
+stanza==1.9.2
+decorator==5.1.1
+seaborn==0.13.0
+sentencepiece==0.2.0
+PyYAML==6.0.1
+black==24.8.0
+protobuf==4.25.1
+pickleshare==0.7.5
+peft==0.13.0
+triton==2.0.0
+nvidia-cuda-runtime-cu11==11.7.99
+Jinja2==3.1.2
+nvidia-cusolver-cu11==11.4.0.1
+executing==1.2.0
+jupyter_client==8.1.0
+pluggy==1.3.0
+cmake==3.30.3
+pytz==2023.3.post1
+aiohappyeyeballs==2.4.2
+kiwisolver==1.4.5
+py-cpuinfo==9.0.0
+Pillow==10.1.0
+ptyprocess==0.7.0
+importlib_resources==6.4.5
+GitPython==3.1.43
+importlib-metadata==6.0.0
+iniconfig==2.0.0
+scikit-learn==1.3.2
+exceptiongroup==1.1.0
+networkx==2.8.6
+accelerate==1.0.0
+nltk==3.8.1
+shutilwhich==1.1.0
+fonttools==4.45.1
+future==0.18.3
+aiohttp==3.10.6
+wcwidth==0.2.5
+idna==3.6
+filelock==3.12.2
+pathspec==0.12.1
+jupyter_core==5.1.0
+lit==18.1.8
+nvidia-curand-cu11==10.2.10.91
+nvidia-cublas-cu11==11.10.3.66
+nvidia-ml-py==12.560.30
+msgpack==1.1.0
+python-dateutil==2.8.2
+blessed==1.20.0
+packaging==23.0
+gitdb==4.0.11
+yarl==1.13.0
+emoji==2.8.0
+tzdata==2023.3
+cycler==0.12.1
+tornado==6.2
+backcall==0.2.0
+plotnine==0.12.4
+ninja==1.11.1.1
+latex==0.7.0
+wandb==0.18.5
+setproctitle==1.3.3
+threadpoolctl==3.2.0
+requests==2.32.3
+pyparsing==3.1.1
+smmap==5.0.1
+pyzmq==23.0.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+matplotlib-inline==0.1.6
+latexcodec==1.0.0
+ipython==8.0.0
+patsy==0.5.3
+contourpy==1.2.0
+multidict==6.1.0
+mizani==0.9.3
+urllib3==2.1.0
+tokenizers==0.20.0
+MarkupSafe==2.1.2
+pip==24.2
+pexpect==4.8.0
+tqdm==4.66.5
+jedi==0.18.2
+pydantic_core==2.23.4
+tempdir==0.7.1
+mpmath==1.2.1
+setuptools==72.1.0
+pytest==7.4.3
+pure-eval==0.2.2
+psutil==5.9.1
+comm==0.1.2
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+regex==2023.10.3
+platformdirs==2.5.2
+asttokens==2.2.1
+torch==2.0.0
+nvidia-nccl-cu11==2.14.3
+xxhash==3.5.0

wandb/run-20241030_013339-pahr4hk1/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-10-30T05:33:39.715818Z",
+  "args":  [
+    "--perturbation",
+    "reverse_full",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "7",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1710081847296"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241030_013339-pahr4hk1/logs/debug-internal.log

@@ -0,0 +1,8 @@
+{"time":"2024-10-30T01:33:39.718219018-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-10-30T01:33:39.718234308-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_013339-pahr4hk1/logs/debug-core.log"}
+{"time":"2024-10-30T01:33:39.825323399-04:00","level":"INFO","msg":"created new stream","id":"pahr4hk1"}
+{"time":"2024-10-30T01:33:39.825359099-04:00","level":"INFO","msg":"stream: started","id":"pahr4hk1"}
+{"time":"2024-10-30T01:33:39.825400679-04:00","level":"INFO","msg":"sender: started","stream_id":"pahr4hk1"}
+{"time":"2024-10-30T01:33:39.825385829-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"pahr4hk1"}}
+{"time":"2024-10-30T01:33:39.825383059-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"pahr4hk1"}}
+{"time":"2024-10-30T01:33:39.990823292-04:00","level":"INFO","msg":"Starting system monitor"}

wandb/run-20241030_013339-pahr4hk1/logs/debug.log

@@ -0,0 +1,26 @@
+2024-10-30 01:33:39,713 INFO    MainThread:337258 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-10-30 01:33:39,714 INFO    MainThread:337258 [wandb_setup.py:_flush():79] Configure stats pid to 337258
+2024-10-30 01:33:39,714 INFO    MainThread:337258 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-10-30 01:33:39,714 INFO    MainThread:337258 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-10-30 01:33:39,714 INFO    MainThread:337258 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-30 01:33:39,714 INFO    MainThread:337258 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-30 01:33:39,714 INFO    MainThread:337258 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-10-30 01:33:39,714 INFO    MainThread:337258 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-30 01:33:39,714 INFO    MainThread:337258 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_013339-pahr4hk1/logs/debug.log
+2024-10-30 01:33:39,714 INFO    MainThread:337258 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_013339-pahr4hk1/logs/debug-internal.log
+2024-10-30 01:33:39,714 INFO    MainThread:337258 [wandb_init.py:init():621] calling init triggers
+2024-10-30 01:33:39,714 INFO    MainThread:337258 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-10-30 01:33:39,714 INFO    MainThread:337258 [wandb_init.py:init():671] starting backend
+2024-10-30 01:33:39,714 INFO    MainThread:337258 [wandb_init.py:init():675] sending inform_init request
+2024-10-30 01:33:39,715 INFO    MainThread:337258 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-30 01:33:39,715 INFO    MainThread:337258 [wandb_init.py:init():688] backend started and connected
+2024-10-30 01:33:39,718 INFO    MainThread:337258 [wandb_init.py:init():783] updated telemetry
+2024-10-30 01:33:39,748 INFO    MainThread:337258 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-10-30 01:33:39,986 INFO    MainThread:337258 [wandb_init.py:init():867] starting run threads in backend
+2024-10-30 01:33:40,076 INFO    MainThread:337258 [wandb_run.py:_console_start():2463] atexit reg
+2024-10-30 01:33:40,076 INFO    MainThread:337258 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-10-30 01:33:40,076 INFO    MainThread:337258 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-10-30 01:33:40,076 INFO    MainThread:337258 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-10-30 01:33:40,077 INFO    MainThread:337258 [wandb_init.py:init():911] run started, returning control to user process
+2024-10-30 01:33:40,078 INFO    MainThread:337258 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_full', 'train_set': '10M', 'batch_size': 3, 'epoch': 7, 'seed': 0}

wandb/run-20241030_112852-av3r7rx8/run-av3r7rx8.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82826fe5130b74ba5b11ab4c02aeb2ce26500de1060a76c495fae27c1f928f5f
+size 14185176

wandb/run-20241030_112852-mfvd6tgw/files/config.yaml

@@ -0,0 +1,48 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.9.19
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "3":
+                - 2
+                - 13
+                - 23
+                - 55
+            "4": 3.9.19
+            "5": 0.18.5
+            "6": 4.45.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+batch_size:
+    value: 3
+epoch:
+    value: 3
+perturbation:
+    value: reverse_control
+seed:
+    value: 0
+train_set:
+    value: 10M

wandb/run-20241030_112852-mfvd6tgw/files/output.log

@@ -0,0 +1,17 @@
+Downloading shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [02:08<00:00, 64.23s/it]
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:04<00:00,  2.16s/it]
+generation_config.json: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 185/185 [00:00<00:00, 57.7kB/s]
+Map: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 18140/18140 [00:49<00:00, 367.87 examples/s]
+tokenized_valid: Dataset({
+    features: ['input_ids', 'attention_mask'],
+    num_rows: 600
+})
+/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
+  warnings.warn(
+[2024-10-30 11:31:57,082] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2024-10-30 11:32:05,240] [INFO] [comm.py:652:init_distributed] cdb=None
+Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination
+Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...
+Loading extension module cpu_adam...
+Time to load cpu_adam op: 4.680240869522095 seconds
+wandb: WARNING Fatal error while uploading data. Some run data will not be synced, but it will still be written to disk. Use `wandb sync` at the end of the run to try uploading.

wandb/run-20241030_112852-mfvd6tgw/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-10-30T15:28:52.868095Z",
+  "args":  [
+    "--perturbation",
+    "reverse_control",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "3",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1710831611904"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241030_112852-mfvd6tgw/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":23503}}

wandb/run-20241030_112852-mfvd6tgw/logs/debug-internal.log

@@ -0,0 +1,108 @@
+{"time":"2024-10-30T11:28:52.871579904-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-10-30T11:28:52.871600584-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_112852-mfvd6tgw/logs/debug-core.log"}
+{"time":"2024-10-30T11:28:52.978841939-04:00","level":"INFO","msg":"created new stream","id":"mfvd6tgw"}
+{"time":"2024-10-30T11:28:52.978902739-04:00","level":"INFO","msg":"stream: started","id":"mfvd6tgw"}
+{"time":"2024-10-30T11:28:52.978926819-04:00","level":"INFO","msg":"sender: started","stream_id":"mfvd6tgw"}
+{"time":"2024-10-30T11:28:52.97893628-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"mfvd6tgw"}}
+{"time":"2024-10-30T11:28:52.978909109-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"mfvd6tgw"}}
+{"time":"2024-10-30T11:28:53.155627531-04:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-10-30T14:03:08.474581851-04:00","level":"ERROR","msg":"HTTP error","status":404,"method":"POST","url":"https://api.wandb.ai/files/yaning1001-dartmouth-college/impossible_llm_reverse/mfvd6tgw/file_stream"}
+{"time":"2024-10-30T14:03:08.487108589-04:00","level":"ERROR+4","msg":"filestream: fatal error: filestream: failed to upload: 404 Not Found path=files/yaning1001-dartmouth-college/impossible_llm_reverse/mfvd6tgw/file_stream: {\"error\":\"run impossible_llm_reverse/mfvd6tgw not found while streaming file\"}"}
+{"time":"2024-10-30T15:29:42.382518231-04:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-10-30T18:00:35.96830223-04:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-10-30T18:00:35.984486769-04:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-10-30T18:00:36.012170229-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:00:36.969090118-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":1.01417858,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:00:38.26382741-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:00:42.839562556-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:00:52.373625385-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:01:08.798689764-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:01:37.002153025-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":61.047236517,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:01:48.282746193-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:02:37.021865642-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":121.066947764,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:02:48.335092366-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:03:37.04344863-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":181.088531722,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:03:48.393221692-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:04:37.067486978-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":241.11257044,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:04:48.448890729-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:05:37.08852491-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":301.133606772,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:05:48.502838457-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:06:37.117047582-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":361.162133104,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:06:48.554665436-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:07:37.139331089-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":421.184416131,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:07:48.61386156-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:08:37.166847552-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":481.211928004,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:08:48.666190809-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:09:37.194996856-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":541.240081368,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:09:48.721077294-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:10:35.955456557-04:00","level":"WARN","msg":"sender: taking a long time","seconds":600.000428857,"work":"WorkRecord(*service_go_proto.Record_Telemetry); Control(connection_id:\"127.0.0.1:35326\")"}
+{"time":"2024-10-30T18:10:37.216651416-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":601.261725058,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:10:48.773966229-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:11:37.244172578-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":661.28925569,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:11:48.824895668-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:12:37.298658553-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":721.343740985,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:12:48.922946274-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:13:37.319508977-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":781.364593549,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:13:48.976226497-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:14:37.34947864-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":841.394560982,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:14:49.026538556-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:15:37.371409433-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":901.416490594,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:15:49.077219977-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:16:37.394261418-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":961.43934326,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:16:49.129011563-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:16:49.129142475-04:00","level":"ERROR","msg":"sender: sendConfig:","error":"api: failed sending: POST https://api.wandb.ai/graphql giving up after 21 attempt(s)"}
+{"time":"2024-10-30T18:16:49.129310967-04:00","level":"INFO","msg":"sender: succeeded after taking longer than expected","seconds":973.174329898,"work":"WorkRecord(*service_go_proto.Record_Telemetry); Control(connection_id:\"127.0.0.1:35326\")"}
+{"time":"2024-10-30T18:16:49.185435119-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:16:49.231279214-04:00","level":"ERROR","msg":"HTTP error","status":404,"method":"POST","url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:16:49.231344485-04:00","level":"ERROR","msg":"runfiles: CreateRunFiles returned error: returned error 404 Not Found: {\"errors\":[{\"message\":\"run impossible_llm_reverse/mfvd6tgw not found during createRunFiles\",\"path\":[\"createRunFiles\"]}],\"data\":{\"createRunFiles\":null}}"}
+{"time":"2024-10-30T18:16:51.551546918-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:16:56.224687098-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:17:04.811694165-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:17:23.782790961-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:17:37.423013569-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":48.293350338,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:18:01.969943103-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:18:37.444499646-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":108.314835984,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:19:02.031162287-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:19:37.46831367-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":168.338651879,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:20:02.086803086-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:20:37.49484466-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":228.365184589,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:21:02.14808835-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:21:37.518777326-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":288.389114975,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:22:02.208668814-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:22:37.542083025-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":348.412418624,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:23:02.265589282-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:23:37.568249482-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":408.438587131,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:24:02.332185182-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:24:37.59016655-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":468.460504129,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:25:02.383191439-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:25:37.611919348-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":528.482255227,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:26:02.439712218-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:26:37.63751757-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":588.507844589,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:26:49.130687726-04:00","level":"WARN","msg":"sender: taking a long time","seconds":600.00065645,"work":"WorkRecord(*service_go_proto.Request_Defer); Control(local:true  always_send:true)"}
+{"time":"2024-10-30T18:27:02.49576742-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:27:37.658073199-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":648.528411148,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:28:02.556603286-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:28:37.678804315-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":708.549129014,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:29:02.616108339-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:29:37.705580303-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":768.575917741,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:30:02.669248598-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:30:37.73008217-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":828.600419649,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:31:02.724511612-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:31:37.753990664-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":888.624324932,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:32:02.784891698-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:32:37.778640619-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run config","runtime_seconds":948.648979298,"error_status":"retrying HTTP 409 Conflict"}],"total_operations":1}}
+{"time":"2024-10-30T18:33:02.873762515-04:00","level":"INFO","msg":"api: retrying HTTP error","status":409,"url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:33:02.873850586-04:00","level":"ERROR","msg":"sender: sendConfig:","error":"api: failed sending: POST https://api.wandb.ai/graphql giving up after 21 attempt(s)"}
+{"time":"2024-10-30T18:33:02.874229689-04:00","level":"INFO","msg":"sender: succeeded after taking longer than expected","seconds":973.744352185,"work":"WorkRecord(*service_go_proto.Request_Defer); Control(local:true  always_send:true)"}
+{"time":"2024-10-30T18:33:02.972769976-04:00","level":"ERROR","msg":"HTTP error","status":404,"method":"POST","url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:33:02.972826436-04:00","level":"ERROR","msg":"runfiles: CreateRunFiles returned error: returned error 404 Not Found: {\"errors\":[{\"message\":\"run impossible_llm_reverse/mfvd6tgw not found during createRunFiles\",\"path\":[\"createRunFiles\"]}],\"data\":{\"createRunFiles\":null}}"}
+{"time":"2024-10-30T18:33:03.110980965-04:00","level":"ERROR","msg":"HTTP error","status":404,"method":"POST","url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:33:03.111192917-04:00","level":"ERROR","msg":"sender: failed to save job artifact: ArtifactSaver.createManifest: returned error 404 Not Found: {\"errors\":[{\"message\":\"failed to find run impossible_llm_reverse/mfvd6tgw\",\"path\":[\"createArtifactManifest\"]}],\"data\":{\"createArtifactManifest\":null}}"}
+{"time":"2024-10-30T18:33:03.162250522-04:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-10-30T18:33:03.212467488-04:00","level":"ERROR","msg":"HTTP error","status":404,"method":"POST","url":"https://api.wandb.ai/graphql"}
+{"time":"2024-10-30T18:33:03.212514949-04:00","level":"ERROR","msg":"runfiles: CreateRunFiles returned error: returned error 404 Not Found: {\"errors\":[{\"message\":\"run impossible_llm_reverse/mfvd6tgw not found during createRunFiles\",\"path\":[\"createRunFiles\"]}],\"data\":{\"createRunFiles\":null}}"}
+{"time":"2024-10-30T18:33:04.172896443-04:00","level":"INFO","msg":"stream: closing","id":"mfvd6tgw"}
+{"time":"2024-10-30T18:33:04.172936993-04:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"mfvd6tgw"}}
+{"time":"2024-10-30T18:33:04.172965223-04:00","level":"INFO","msg":"sender: closed","stream_id":"mfvd6tgw"}
+{"time":"2024-10-30T18:33:04.172958023-04:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"mfvd6tgw"}}
+{"time":"2024-10-30T18:33:04.173060174-04:00","level":"INFO","msg":"stream: closed","id":"mfvd6tgw"}

wandb/run-20241030_112852-mfvd6tgw/logs/debug.log

@@ -0,0 +1,33 @@
+2024-10-30 11:28:52,866 INFO    MainThread:367766 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-10-30 11:28:52,866 INFO    MainThread:367766 [wandb_setup.py:_flush():79] Configure stats pid to 367766
+2024-10-30 11:28:52,866 INFO    MainThread:367766 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-10-30 11:28:52,866 INFO    MainThread:367766 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-10-30 11:28:52,866 INFO    MainThread:367766 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-30 11:28:52,866 INFO    MainThread:367766 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-30 11:28:52,866 INFO    MainThread:367766 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-10-30 11:28:52,866 INFO    MainThread:367766 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-30 11:28:52,866 INFO    MainThread:367766 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_112852-mfvd6tgw/logs/debug.log
+2024-10-30 11:28:52,866 INFO    MainThread:367766 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_112852-mfvd6tgw/logs/debug-internal.log
+2024-10-30 11:28:52,866 INFO    MainThread:367766 [wandb_init.py:init():621] calling init triggers
+2024-10-30 11:28:52,866 INFO    MainThread:367766 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-10-30 11:28:52,866 INFO    MainThread:367766 [wandb_init.py:init():671] starting backend
+2024-10-30 11:28:52,866 INFO    MainThread:367766 [wandb_init.py:init():675] sending inform_init request
+2024-10-30 11:28:52,867 INFO    MainThread:367766 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-30 11:28:52,867 INFO    MainThread:367766 [wandb_init.py:init():688] backend started and connected
+2024-10-30 11:28:52,871 INFO    MainThread:367766 [wandb_init.py:init():783] updated telemetry
+2024-10-30 11:28:52,896 INFO    MainThread:367766 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-10-30 11:28:53,152 INFO    MainThread:367766 [wandb_init.py:init():867] starting run threads in backend
+2024-10-30 11:28:53,239 INFO    MainThread:367766 [wandb_run.py:_console_start():2463] atexit reg
+2024-10-30 11:28:53,239 INFO    MainThread:367766 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-10-30 11:28:53,239 INFO    MainThread:367766 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-10-30 11:28:53,239 INFO    MainThread:367766 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-10-30 11:28:53,240 INFO    MainThread:367766 [wandb_init.py:init():911] run started, returning control to user process
+2024-10-30 11:28:53,240 INFO    MainThread:367766 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_control', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0}
+2024-10-30 18:00:35,945 INFO    MainThread:367766 [wandb_run.py:_finish():2158] finishing run yaning1001-dartmouth-college/impossible_llm_reverse/mfvd6tgw
+2024-10-30 18:00:35,954 INFO    MainThread:367766 [wandb_run.py:_atexit_cleanup():2426] got exitcode: 0
+2024-10-30 18:00:35,955 INFO    MainThread:367766 [wandb_run.py:_restore():2408] restore
+2024-10-30 18:00:35,955 INFO    MainThread:367766 [wandb_run.py:_restore():2414] restore done
+2024-10-30 18:33:04,165 INFO    MainThread:367766 [wandb_run.py:_footer_history_summary_info():3975] rendering history
+2024-10-30 18:33:04,166 INFO    MainThread:367766 [wandb_run.py:_footer_history_summary_info():4007] rendering summary
+2024-10-30 18:33:04,172 INFO    MainThread:367766 [wandb_run.py:_footer_sync_info():3934] logging synced files

wandb/run-20241030_233740-np98q8en/run-np98q8en.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a85d468fd545c171fcd433a279c3cfc6210c775149911b0f456f66e43c4b7c64
+size 851968

wandb/run-20241031_000839-acpkxm8c/files/output.log

@@ -0,0 +1,13 @@
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:18<00:00,  9.31s/it]
+tokenized_valid: Dataset({
+    features: ['input_ids', 'attention_mask'],
+    num_rows: 600
+})
+/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
+  warnings.warn(
+[2024-10-31 00:09:00,175] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2024-10-31 00:09:10,031] [INFO] [comm.py:652:init_distributed] cdb=None
+Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination
+Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...
+Loading extension module cpu_adam...
+Time to load cpu_adam op: 5.321483373641968 seconds

wandb/run-20241031_000839-acpkxm8c/files/requirements.txt

@@ -0,0 +1,147 @@
+funcsigs==1.0.2
+sentry-sdk==2.17.0
+multiprocess==0.70.16
+numpy==1.26.2
+pluralizer==1.2.0
+debugpy==1.6.7
+nvidia-cudnn-cu11==8.5.0.96
+deepspeed==0.15.2
+data==0.4
+pandas==2.1.3
+tomli==2.0.1
+charset-normalizer==3.3.2
+attrs==24.2.0
+aiosignal==1.3.1
+fsspec==2023.10.0
+nvidia-cusparse-cu11==11.7.4.91
+zipp==3.12.0
+mypy-extensions==1.0.0
+datasets==3.0.1
+joblib==1.3.2
+hjson==3.1.0
+traitlets==5.7.1
+stack-data==0.6.0
+transformers==4.45.1
+sympy==1.11.1
+Pygments==2.15.0
+docker-pycreds==0.4.0
+dill==0.3.8
+wheel==0.44.0
+prompt-toolkit==3.0.30
+parso==0.8.3
+ipykernel==6.23.1
+pyarrow==17.0.0
+certifi==2023.11.17
+nvidia-cufft-cu11==10.9.0.58
+six==1.16.0
+pydantic==2.9.2
+click==8.1.7
+nest-asyncio==1.5.6
+gmpy2==2.1.0
+matplotlib==3.8.2
+scipy==1.11.4
+typing_extensions==4.12.2
+statsmodels==0.14.0
+huggingface-hub==0.25.0
+frozenlist==1.4.1
+gpustat==1.1.1
+nvidia-nvtx-cu11==11.7.91
+safetensors==0.4.5
+stanza==1.9.2
+decorator==5.1.1
+seaborn==0.13.0
+sentencepiece==0.2.0
+PyYAML==6.0.1
+black==24.8.0
+protobuf==4.25.1
+pickleshare==0.7.5
+peft==0.13.0
+triton==2.0.0
+nvidia-cuda-runtime-cu11==11.7.99
+Jinja2==3.1.2
+nvidia-cusolver-cu11==11.4.0.1
+executing==1.2.0
+jupyter_client==8.1.0
+pluggy==1.3.0
+cmake==3.30.3
+pytz==2023.3.post1
+aiohappyeyeballs==2.4.2
+kiwisolver==1.4.5
+py-cpuinfo==9.0.0
+Pillow==10.1.0
+ptyprocess==0.7.0
+importlib_resources==6.4.5
+GitPython==3.1.43
+importlib-metadata==6.0.0
+iniconfig==2.0.0
+scikit-learn==1.3.2
+exceptiongroup==1.1.0
+networkx==2.8.6
+accelerate==1.0.0
+nltk==3.8.1
+shutilwhich==1.1.0
+fonttools==4.45.1
+future==0.18.3
+aiohttp==3.10.6
+wcwidth==0.2.5
+idna==3.6
+filelock==3.12.2
+pathspec==0.12.1
+jupyter_core==5.1.0
+lit==18.1.8
+nvidia-curand-cu11==10.2.10.91
+nvidia-cublas-cu11==11.10.3.66
+nvidia-ml-py==12.560.30
+msgpack==1.1.0
+python-dateutil==2.8.2
+blessed==1.20.0
+packaging==23.0
+gitdb==4.0.11
+yarl==1.13.0
+emoji==2.8.0
+tzdata==2023.3
+cycler==0.12.1
+tornado==6.2
+backcall==0.2.0
+plotnine==0.12.4
+ninja==1.11.1.1
+latex==0.7.0
+wandb==0.18.5
+setproctitle==1.3.3
+threadpoolctl==3.2.0
+requests==2.32.3
+pyparsing==3.1.1
+smmap==5.0.1
+pyzmq==23.0.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+matplotlib-inline==0.1.6
+latexcodec==1.0.0
+ipython==8.0.0
+patsy==0.5.3
+contourpy==1.2.0
+multidict==6.1.0
+mizani==0.9.3
+urllib3==2.1.0
+tokenizers==0.20.0
+MarkupSafe==2.1.2
+pip==24.2
+pexpect==4.8.0
+tqdm==4.66.5
+jedi==0.18.2
+pydantic_core==2.23.4
+tempdir==0.7.1
+mpmath==1.2.1
+setuptools==72.1.0
+pytest==7.4.3
+pure-eval==0.2.2
+psutil==5.9.1
+comm==0.1.2
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+regex==2023.10.3
+platformdirs==2.5.2
+asttokens==2.2.1
+torch==2.0.0
+nvidia-nccl-cu11==2.14.3
+xxhash==3.5.0

wandb/run-20241031_000839-acpkxm8c/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-10-31T04:08:39.164895Z",
+  "args":  [
+    "--perturbation",
+    "reverse_full",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "6",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1727270539264"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241031_000839-acpkxm8c/logs/debug-internal.log

@@ -0,0 +1,8 @@
+{"time":"2024-10-31T00:08:39.166868509-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-10-31T00:08:39.166882039-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_000839-acpkxm8c/logs/debug-core.log"}
+{"time":"2024-10-31T00:08:39.272997415-04:00","level":"INFO","msg":"created new stream","id":"acpkxm8c"}
+{"time":"2024-10-31T00:08:39.273045656-04:00","level":"INFO","msg":"stream: started","id":"acpkxm8c"}
+{"time":"2024-10-31T00:08:39.273081016-04:00","level":"INFO","msg":"sender: started","stream_id":"acpkxm8c"}
+{"time":"2024-10-31T00:08:39.273075736-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"acpkxm8c"}}
+{"time":"2024-10-31T00:08:39.273091156-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"acpkxm8c"}}
+{"time":"2024-10-31T00:08:39.475372241-04:00","level":"INFO","msg":"Starting system monitor"}

wandb/run-20241031_000839-acpkxm8c/logs/debug.log

@@ -0,0 +1,26 @@
+2024-10-31 00:08:39,162 INFO    MainThread:477299 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-10-31 00:08:39,163 INFO    MainThread:477299 [wandb_setup.py:_flush():79] Configure stats pid to 477299
+2024-10-31 00:08:39,163 INFO    MainThread:477299 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-10-31 00:08:39,163 INFO    MainThread:477299 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-10-31 00:08:39,163 INFO    MainThread:477299 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-31 00:08:39,163 INFO    MainThread:477299 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-31 00:08:39,163 INFO    MainThread:477299 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-10-31 00:08:39,163 INFO    MainThread:477299 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-31 00:08:39,163 INFO    MainThread:477299 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_000839-acpkxm8c/logs/debug.log
+2024-10-31 00:08:39,163 INFO    MainThread:477299 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_000839-acpkxm8c/logs/debug-internal.log
+2024-10-31 00:08:39,163 INFO    MainThread:477299 [wandb_init.py:init():621] calling init triggers
+2024-10-31 00:08:39,163 INFO    MainThread:477299 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-10-31 00:08:39,163 INFO    MainThread:477299 [wandb_init.py:init():671] starting backend
+2024-10-31 00:08:39,163 INFO    MainThread:477299 [wandb_init.py:init():675] sending inform_init request
+2024-10-31 00:08:39,164 INFO    MainThread:477299 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-31 00:08:39,164 INFO    MainThread:477299 [wandb_init.py:init():688] backend started and connected
+2024-10-31 00:08:39,167 INFO    MainThread:477299 [wandb_init.py:init():783] updated telemetry
+2024-10-31 00:08:39,194 INFO    MainThread:477299 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-10-31 00:08:39,472 INFO    MainThread:477299 [wandb_init.py:init():867] starting run threads in backend
+2024-10-31 00:08:39,571 INFO    MainThread:477299 [wandb_run.py:_console_start():2463] atexit reg
+2024-10-31 00:08:39,571 INFO    MainThread:477299 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-10-31 00:08:39,571 INFO    MainThread:477299 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-10-31 00:08:39,571 INFO    MainThread:477299 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-10-31 00:08:39,572 INFO    MainThread:477299 [wandb_init.py:init():911] run started, returning control to user process
+2024-10-31 00:08:39,572 INFO    MainThread:477299 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_full', 'train_set': '10M', 'batch_size': 3, 'epoch': 6, 'seed': 0, 'lr': 1e-05}

wandb/run-20241101_012733-3tsgnm2p/run-3tsgnm2p.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:157ac6fcdd37aa5de6b664b2b512542501c187eda74f9151bcca4bd636d73df0
+size 1015808

wandb/run-20241101_200502-7hem25r3/files/output.log

@@ -0,0 +1 @@
+Loading checkpoint shards:   0%|                                                                                                                                                        | 0/2 [00:00<?, ?it/s]

wandb/run-20241101_200502-7hem25r3/files/requirements.txt

@@ -0,0 +1,147 @@
+funcsigs==1.0.2
+sentry-sdk==2.17.0
+multiprocess==0.70.16
+numpy==1.26.2
+pluralizer==1.2.0
+debugpy==1.6.7
+nvidia-cudnn-cu11==8.5.0.96
+deepspeed==0.15.2
+data==0.4
+pandas==2.1.3
+tomli==2.0.1
+charset-normalizer==3.3.2
+attrs==24.2.0
+aiosignal==1.3.1
+fsspec==2023.10.0
+nvidia-cusparse-cu11==11.7.4.91
+zipp==3.12.0
+mypy-extensions==1.0.0
+datasets==3.0.1
+joblib==1.3.2
+hjson==3.1.0
+traitlets==5.7.1
+stack-data==0.6.0
+transformers==4.45.1
+sympy==1.11.1
+Pygments==2.15.0
+docker-pycreds==0.4.0
+dill==0.3.8
+wheel==0.44.0
+prompt-toolkit==3.0.30
+parso==0.8.3
+ipykernel==6.23.1
+pyarrow==17.0.0
+certifi==2023.11.17
+nvidia-cufft-cu11==10.9.0.58
+six==1.16.0
+pydantic==2.9.2
+click==8.1.7
+nest-asyncio==1.5.6
+gmpy2==2.1.0
+matplotlib==3.8.2
+scipy==1.11.4
+typing_extensions==4.12.2
+statsmodels==0.14.0
+huggingface-hub==0.25.0
+frozenlist==1.4.1
+gpustat==1.1.1
+nvidia-nvtx-cu11==11.7.91
+safetensors==0.4.5
+stanza==1.9.2
+decorator==5.1.1
+seaborn==0.13.0
+sentencepiece==0.2.0
+PyYAML==6.0.1
+black==24.8.0
+protobuf==4.25.1
+pickleshare==0.7.5
+peft==0.13.0
+triton==2.0.0
+nvidia-cuda-runtime-cu11==11.7.99
+Jinja2==3.1.2
+nvidia-cusolver-cu11==11.4.0.1
+executing==1.2.0
+jupyter_client==8.1.0
+pluggy==1.3.0
+cmake==3.30.3
+pytz==2023.3.post1
+aiohappyeyeballs==2.4.2
+kiwisolver==1.4.5
+py-cpuinfo==9.0.0
+Pillow==10.1.0
+ptyprocess==0.7.0
+importlib_resources==6.4.5
+GitPython==3.1.43
+importlib-metadata==6.0.0
+iniconfig==2.0.0
+scikit-learn==1.3.2
+exceptiongroup==1.1.0
+networkx==2.8.6
+accelerate==1.0.0
+nltk==3.8.1
+shutilwhich==1.1.0
+fonttools==4.45.1
+future==0.18.3
+aiohttp==3.10.6
+wcwidth==0.2.5
+idna==3.6
+filelock==3.12.2
+pathspec==0.12.1
+jupyter_core==5.1.0
+lit==18.1.8
+nvidia-curand-cu11==10.2.10.91
+nvidia-cublas-cu11==11.10.3.66
+nvidia-ml-py==12.560.30
+msgpack==1.1.0
+python-dateutil==2.8.2
+blessed==1.20.0
+packaging==23.0
+gitdb==4.0.11
+yarl==1.13.0
+emoji==2.8.0
+tzdata==2023.3
+cycler==0.12.1
+tornado==6.2
+backcall==0.2.0
+plotnine==0.12.4
+ninja==1.11.1.1
+latex==0.7.0
+wandb==0.18.5
+setproctitle==1.3.3
+threadpoolctl==3.2.0
+requests==2.32.3
+pyparsing==3.1.1
+smmap==5.0.1
+pyzmq==23.0.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+matplotlib-inline==0.1.6
+latexcodec==1.0.0
+ipython==8.0.0
+patsy==0.5.3
+contourpy==1.2.0
+multidict==6.1.0
+mizani==0.9.3
+urllib3==2.1.0
+tokenizers==0.20.0
+MarkupSafe==2.1.2
+pip==24.2
+pexpect==4.8.0
+tqdm==4.66.5
+jedi==0.18.2
+pydantic_core==2.23.4
+tempdir==0.7.1
+mpmath==1.2.1
+setuptools==72.1.0
+pytest==7.4.3
+pure-eval==0.2.2
+psutil==5.9.1
+comm==0.1.2
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+regex==2023.10.3
+platformdirs==2.5.2
+asttokens==2.2.1
+torch==2.0.0
+nvidia-nccl-cu11==2.14.3
+xxhash==3.5.0

wandb/run-20241101_200502-7hem25r3/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-11-02T00:05:02.628844Z",
+  "args":  [
+    "--perturbation",
+    "shuffle_nondeterministic",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "3",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1754801463296"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241101_200502-7hem25r3/logs/debug-internal.log

@@ -0,0 +1,8 @@
+{"time":"2024-11-01T20:05:02.633735347-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-11-01T20:05:02.633752607-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_200502-7hem25r3/logs/debug-core.log"}
+{"time":"2024-11-01T20:05:02.844674605-04:00","level":"INFO","msg":"created new stream","id":"7hem25r3"}
+{"time":"2024-11-01T20:05:02.844749605-04:00","level":"INFO","msg":"stream: started","id":"7hem25r3"}
+{"time":"2024-11-01T20:05:02.844804386-04:00","level":"INFO","msg":"sender: started","stream_id":"7hem25r3"}
+{"time":"2024-11-01T20:05:02.844777586-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"7hem25r3"}}
+{"time":"2024-11-01T20:05:02.844794496-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"7hem25r3"}}
+{"time":"2024-11-01T20:05:03.055749493-04:00","level":"INFO","msg":"Starting system monitor"}

wandb/run-20241101_200502-7hem25r3/logs/debug.log

@@ -0,0 +1,26 @@
+2024-11-01 20:05:02,626 INFO    MainThread:869512 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-11-01 20:05:02,626 INFO    MainThread:869512 [wandb_setup.py:_flush():79] Configure stats pid to 869512
+2024-11-01 20:05:02,626 INFO    MainThread:869512 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-11-01 20:05:02,626 INFO    MainThread:869512 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-11-01 20:05:02,626 INFO    MainThread:869512 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-11-01 20:05:02,626 INFO    MainThread:869512 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-11-01 20:05:02,626 INFO    MainThread:869512 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-11-01 20:05:02,626 INFO    MainThread:869512 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-11-01 20:05:02,626 INFO    MainThread:869512 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_200502-7hem25r3/logs/debug.log
+2024-11-01 20:05:02,626 INFO    MainThread:869512 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_200502-7hem25r3/logs/debug-internal.log
+2024-11-01 20:05:02,626 INFO    MainThread:869512 [wandb_init.py:init():621] calling init triggers
+2024-11-01 20:05:02,627 INFO    MainThread:869512 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-11-01 20:05:02,627 INFO    MainThread:869512 [wandb_init.py:init():671] starting backend
+2024-11-01 20:05:02,627 INFO    MainThread:869512 [wandb_init.py:init():675] sending inform_init request
+2024-11-01 20:05:02,628 INFO    MainThread:869512 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-11-01 20:05:02,628 INFO    MainThread:869512 [wandb_init.py:init():688] backend started and connected
+2024-11-01 20:05:02,631 INFO    MainThread:869512 [wandb_init.py:init():783] updated telemetry
+2024-11-01 20:05:02,651 INFO    MainThread:869512 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-11-01 20:05:03,053 INFO    MainThread:869512 [wandb_init.py:init():867] starting run threads in backend
+2024-11-01 20:05:03,140 INFO    MainThread:869512 [wandb_run.py:_console_start():2463] atexit reg
+2024-11-01 20:05:03,140 INFO    MainThread:869512 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-11-01 20:05:03,140 INFO    MainThread:869512 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-11-01 20:05:03,140 INFO    MainThread:869512 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-11-01 20:05:03,142 INFO    MainThread:869512 [wandb_init.py:init():911] run started, returning control to user process
+2024-11-01 20:05:03,142 INFO    MainThread:869512 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_nondeterministic', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0, 'lr': 5e-06}

wandb/run-20241101_200517-vzv5zg2q/files/config.yaml

@@ -0,0 +1,49 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.9.19
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "3":
+                - 13
+                - 23
+                - 55
+            "4": 3.9.19
+            "5": 0.18.5
+            "6": 4.45.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+batch_size:
+    value: 3
+epoch:
+    value: 3
+lr:
+    value: 5e-06
+perturbation:
+    value: shuffle_nondeterministic
+seed:
+    value: 0
+train_set:
+    value: 10M

wandb/run-20241101_200517-vzv5zg2q/files/output.log

@@ -0,0 +1,34 @@
+Downloading shards:   0%|                                                                                                                                                               | 0/2 [00:07<?, ?it/s]
+Error in sys.excepthook:
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/wandb/sdk/lib/exit_hooks.py", line 52, in exc_handler
+    traceback.print_exception(exc_type, exc, tb)
+KeyboardInterrupt
+
+Original exception was:
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 173, in <module>
+    model = AutoModelForCausalLM.from_pretrained(model_name,
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/models/auto/auto_factory.py", line 564, in from_pretrained
+    return model_class.from_pretrained(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/modeling_utils.py", line 3769, in from_pretrained
+    resolved_archive_file, sharded_metadata = get_checkpoint_shard_files(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/utils/hub.py", line 1098, in get_checkpoint_shard_files
+    cached_filename = cached_file(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/utils/hub.py", line 403, in cached_file
+    resolved_file = hf_hub_download(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/utils/_deprecation.py", line 101, in inner_f
+    return f(*args, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn
+    return fn(*args, **kwargs)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1232, in hf_hub_download
+    return _hf_hub_download_to_cache_dir(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1380, in _hf_hub_download_to_cache_dir
+    with WeakFileLock(lock_path):
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/contextlib.py", line 119, in __enter__
+    return next(self.gen)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/utils/_fixes.py", line 98, in WeakFileLock
+    lock.acquire()
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/filelock/_api.py", line 225, in acquire
+    time.sleep(poll_interval)
+KeyboardInterrupt

wandb/run-20241101_200517-vzv5zg2q/files/requirements.txt

@@ -0,0 +1,147 @@
+funcsigs==1.0.2
+sentry-sdk==2.17.0
+multiprocess==0.70.16
+numpy==1.26.2
+pluralizer==1.2.0
+debugpy==1.6.7
+nvidia-cudnn-cu11==8.5.0.96
+deepspeed==0.15.2
+data==0.4
+pandas==2.1.3
+tomli==2.0.1
+charset-normalizer==3.3.2
+attrs==24.2.0
+aiosignal==1.3.1
+fsspec==2023.10.0
+nvidia-cusparse-cu11==11.7.4.91
+zipp==3.12.0
+mypy-extensions==1.0.0
+datasets==3.0.1
+joblib==1.3.2
+hjson==3.1.0
+traitlets==5.7.1
+stack-data==0.6.0
+transformers==4.45.1
+sympy==1.11.1
+Pygments==2.15.0
+docker-pycreds==0.4.0
+dill==0.3.8
+wheel==0.44.0
+prompt-toolkit==3.0.30
+parso==0.8.3
+ipykernel==6.23.1
+pyarrow==17.0.0
+certifi==2023.11.17
+nvidia-cufft-cu11==10.9.0.58
+six==1.16.0
+pydantic==2.9.2
+click==8.1.7
+nest-asyncio==1.5.6
+gmpy2==2.1.0
+matplotlib==3.8.2
+scipy==1.11.4
+typing_extensions==4.12.2
+statsmodels==0.14.0
+huggingface-hub==0.25.0
+frozenlist==1.4.1
+gpustat==1.1.1
+nvidia-nvtx-cu11==11.7.91
+safetensors==0.4.5
+stanza==1.9.2
+decorator==5.1.1
+seaborn==0.13.0
+sentencepiece==0.2.0
+PyYAML==6.0.1
+black==24.8.0
+protobuf==4.25.1
+pickleshare==0.7.5
+peft==0.13.0
+triton==2.0.0
+nvidia-cuda-runtime-cu11==11.7.99
+Jinja2==3.1.2
+nvidia-cusolver-cu11==11.4.0.1
+executing==1.2.0
+jupyter_client==8.1.0
+pluggy==1.3.0
+cmake==3.30.3
+pytz==2023.3.post1
+aiohappyeyeballs==2.4.2
+kiwisolver==1.4.5
+py-cpuinfo==9.0.0
+Pillow==10.1.0
+ptyprocess==0.7.0
+importlib_resources==6.4.5
+GitPython==3.1.43
+importlib-metadata==6.0.0
+iniconfig==2.0.0
+scikit-learn==1.3.2
+exceptiongroup==1.1.0
+networkx==2.8.6
+accelerate==1.0.0
+nltk==3.8.1
+shutilwhich==1.1.0
+fonttools==4.45.1
+future==0.18.3
+aiohttp==3.10.6
+wcwidth==0.2.5
+idna==3.6
+filelock==3.12.2
+pathspec==0.12.1
+jupyter_core==5.1.0
+lit==18.1.8
+nvidia-curand-cu11==10.2.10.91
+nvidia-cublas-cu11==11.10.3.66
+nvidia-ml-py==12.560.30
+msgpack==1.1.0
+python-dateutil==2.8.2
+blessed==1.20.0
+packaging==23.0
+gitdb==4.0.11
+yarl==1.13.0
+emoji==2.8.0
+tzdata==2023.3
+cycler==0.12.1
+tornado==6.2
+backcall==0.2.0
+plotnine==0.12.4
+ninja==1.11.1.1
+latex==0.7.0
+wandb==0.18.5
+setproctitle==1.3.3
+threadpoolctl==3.2.0
+requests==2.32.3
+pyparsing==3.1.1
+smmap==5.0.1
+pyzmq==23.0.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+matplotlib-inline==0.1.6
+latexcodec==1.0.0
+ipython==8.0.0
+patsy==0.5.3
+contourpy==1.2.0
+multidict==6.1.0
+mizani==0.9.3
+urllib3==2.1.0
+tokenizers==0.20.0
+MarkupSafe==2.1.2
+pip==24.2
+pexpect==4.8.0
+tqdm==4.66.5
+jedi==0.18.2
+pydantic_core==2.23.4
+tempdir==0.7.1
+mpmath==1.2.1
+setuptools==72.1.0
+pytest==7.4.3
+pure-eval==0.2.2
+psutil==5.9.1
+comm==0.1.2
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+regex==2023.10.3
+platformdirs==2.5.2
+asttokens==2.2.1
+torch==2.0.0
+nvidia-nccl-cu11==2.14.3
+xxhash==3.5.0

wandb/run-20241101_200517-vzv5zg2q/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-11-02T00:05:17.600434Z",
+  "args":  [
+    "--perturbation",
+    "shuffle_nondeterministic",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "3",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1754801557504"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241101_200517-vzv5zg2q/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":7}}

wandb/run-20241101_200517-vzv5zg2q/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2024-11-01T20:05:17.602749928-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-11-01T20:05:17.602763728-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_200517-vzv5zg2q/logs/debug-core.log"}
+{"time":"2024-11-01T20:05:17.709188149-04:00","level":"INFO","msg":"created new stream","id":"vzv5zg2q"}
+{"time":"2024-11-01T20:05:17.70922197-04:00","level":"INFO","msg":"stream: started","id":"vzv5zg2q"}
+{"time":"2024-11-01T20:05:17.70926242-04:00","level":"INFO","msg":"sender: started","stream_id":"vzv5zg2q"}
+{"time":"2024-11-01T20:05:17.70924561-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"vzv5zg2q"}}
+{"time":"2024-11-01T20:05:17.709359941-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"vzv5zg2q"}}
+{"time":"2024-11-01T20:05:17.906758123-04:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-11-01T20:05:25.260555442-04:00","level":"INFO","msg":"stream: closing","id":"vzv5zg2q"}
+{"time":"2024-11-01T20:05:25.260678473-04:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-11-01T20:05:25.261811351-04:00","level":"INFO","msg":"Stopped system monitor"}

wandb/run-20241101_200517-vzv5zg2q/logs/debug.log

@@ -0,0 +1,27 @@
+2024-11-01 20:05:17,597 INFO    MainThread:870384 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-11-01 20:05:17,597 INFO    MainThread:870384 [wandb_setup.py:_flush():79] Configure stats pid to 870384
+2024-11-01 20:05:17,597 INFO    MainThread:870384 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-11-01 20:05:17,597 INFO    MainThread:870384 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-11-01 20:05:17,597 INFO    MainThread:870384 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-11-01 20:05:17,597 INFO    MainThread:870384 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-11-01 20:05:17,597 INFO    MainThread:870384 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-11-01 20:05:17,597 INFO    MainThread:870384 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-11-01 20:05:17,597 INFO    MainThread:870384 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_200517-vzv5zg2q/logs/debug.log
+2024-11-01 20:05:17,597 INFO    MainThread:870384 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_200517-vzv5zg2q/logs/debug-internal.log
+2024-11-01 20:05:17,598 INFO    MainThread:870384 [wandb_init.py:init():621] calling init triggers
+2024-11-01 20:05:17,598 INFO    MainThread:870384 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-11-01 20:05:17,598 INFO    MainThread:870384 [wandb_init.py:init():671] starting backend
+2024-11-01 20:05:17,598 INFO    MainThread:870384 [wandb_init.py:init():675] sending inform_init request
+2024-11-01 20:05:17,599 INFO    MainThread:870384 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-11-01 20:05:17,600 INFO    MainThread:870384 [wandb_init.py:init():688] backend started and connected
+2024-11-01 20:05:17,603 INFO    MainThread:870384 [wandb_init.py:init():783] updated telemetry
+2024-11-01 20:05:17,631 INFO    MainThread:870384 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-11-01 20:05:17,903 INFO    MainThread:870384 [wandb_init.py:init():867] starting run threads in backend
+2024-11-01 20:05:17,993 INFO    MainThread:870384 [wandb_run.py:_console_start():2463] atexit reg
+2024-11-01 20:05:17,993 INFO    MainThread:870384 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-11-01 20:05:17,993 INFO    MainThread:870384 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-11-01 20:05:17,993 INFO    MainThread:870384 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-11-01 20:05:17,994 INFO    MainThread:870384 [wandb_init.py:init():911] run started, returning control to user process
+2024-11-01 20:05:17,994 INFO    MainThread:870384 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_nondeterministic', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0, 'lr': 5e-06}
+2024-11-01 20:05:25,260 WARNING MsgRouterThr:870384 [router.py:message_loop():77] message_loop has been closed

wandb/run-20241101_201630-e5gt2fir/files/config.yaml

@@ -0,0 +1,49 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.9.19
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "3":
+                - 13
+                - 23
+                - 55
+            "4": 3.9.19
+            "5": 0.18.5
+            "6": 4.45.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+batch_size:
+    value: 3
+epoch:
+    value: 6
+lr:
+    value: 5e-06
+perturbation:
+    value: shuffle_nodeterministic
+seed:
+    value: 0
+train_set:
+    value: 10M

wandb/run-20241101_201630-e5gt2fir/files/output.log

@@ -0,0 +1,12 @@
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 164, in <module>
+    dataset = load_dataset('babylm_dataset_test.py', name=dataset_name, trust_remote_code=True)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/load.py", line 2074, in load_dataset
+    builder_instance = load_dataset_builder(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/load.py", line 1832, in load_dataset_builder
+    builder_instance: DatasetBuilder = builder_cls(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/builder.py", line 342, in __init__
+    self.config, self.config_id = self._create_builder_config(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/builder.py", line 569, in _create_builder_config
+    raise ValueError(
+ValueError: BuilderConfig 'babylm_shuffle_nodeterministic_10M_seed0' not found. Available: ['babylm_hop_control_10M_seed0', 'babylm_hop_tokens4_10M_seed0', 'babylm_hop_words4_10M_seed0', 'babylm_reverse_control_10M_seed0', 'babylm_reverse_partial_10M_seed0', 'babylm_reverse_full_10M_seed0', 'babylm_shuffle_control_10M_seed0', 'babylm_shuffle_nondeterministic_10M_seed0', 'babylm_shuffle_deterministic21_10M_seed0', 'babylm_shuffle_deterministic57_10M_seed0', 'babylm_shuffle_deterministic84_10M_seed0', 'babylm_shuffle_local3_10M_seed0', 'babylm_shuffle_local5_10M_seed0', 'babylm_shuffle_local10_10M_seed0', 'babylm_shuffle_even_odd_10M_seed0']

wandb/run-20241101_201630-e5gt2fir/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":0}}

wandb/run-20241101_201630-e5gt2fir/logs/debug-internal.log

@@ -0,0 +1,16 @@
+{"time":"2024-11-01T20:16:30.561699495-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-11-01T20:16:30.561714855-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_201630-e5gt2fir/logs/debug-core.log"}
+{"time":"2024-11-01T20:16:30.670004167-04:00","level":"INFO","msg":"created new stream","id":"e5gt2fir"}
+{"time":"2024-11-01T20:16:30.670039398-04:00","level":"INFO","msg":"stream: started","id":"e5gt2fir"}
+{"time":"2024-11-01T20:16:30.670068758-04:00","level":"INFO","msg":"sender: started","stream_id":"e5gt2fir"}
+{"time":"2024-11-01T20:16:30.670067278-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"e5gt2fir"}}
+{"time":"2024-11-01T20:16:30.670100128-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"e5gt2fir"}}
+{"time":"2024-11-01T20:16:30.881477183-04:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-11-01T20:16:30.974886093-04:00","level":"INFO","msg":"stream: closing","id":"e5gt2fir"}
+{"time":"2024-11-01T20:16:30.974923473-04:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-11-01T20:16:30.982511386-04:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-11-01T20:16:31.563553389-04:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-11-01T20:16:31.686184401-04:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"e5gt2fir"}}
+{"time":"2024-11-01T20:16:31.686245181-04:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"e5gt2fir"}}
+{"time":"2024-11-01T20:16:31.686264612-04:00","level":"INFO","msg":"sender: closed","stream_id":"e5gt2fir"}
+{"time":"2024-11-01T20:16:31.686314312-04:00","level":"INFO","msg":"stream: closed","id":"e5gt2fir"}

wandb/run-20241101_201630-e5gt2fir/logs/debug.log

@@ -0,0 +1,27 @@
+2024-11-01 20:16:30,556 INFO    MainThread:874718 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-11-01 20:16:30,557 INFO    MainThread:874718 [wandb_setup.py:_flush():79] Configure stats pid to 874718
+2024-11-01 20:16:30,557 INFO    MainThread:874718 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-11-01 20:16:30,557 INFO    MainThread:874718 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-11-01 20:16:30,557 INFO    MainThread:874718 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-11-01 20:16:30,557 INFO    MainThread:874718 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-11-01 20:16:30,557 INFO    MainThread:874718 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-11-01 20:16:30,557 INFO    MainThread:874718 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-11-01 20:16:30,557 INFO    MainThread:874718 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_201630-e5gt2fir/logs/debug.log
+2024-11-01 20:16:30,557 INFO    MainThread:874718 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_201630-e5gt2fir/logs/debug-internal.log
+2024-11-01 20:16:30,557 INFO    MainThread:874718 [wandb_init.py:init():621] calling init triggers
+2024-11-01 20:16:30,557 INFO    MainThread:874718 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-11-01 20:16:30,557 INFO    MainThread:874718 [wandb_init.py:init():671] starting backend
+2024-11-01 20:16:30,557 INFO    MainThread:874718 [wandb_init.py:init():675] sending inform_init request
+2024-11-01 20:16:30,559 INFO    MainThread:874718 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-11-01 20:16:30,559 INFO    MainThread:874718 [wandb_init.py:init():688] backend started and connected
+2024-11-01 20:16:30,563 INFO    MainThread:874718 [wandb_init.py:init():783] updated telemetry
+2024-11-01 20:16:30,592 INFO    MainThread:874718 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-11-01 20:16:30,878 INFO    MainThread:874718 [wandb_init.py:init():867] starting run threads in backend
+2024-11-01 20:16:30,965 INFO    MainThread:874718 [wandb_run.py:_console_start():2463] atexit reg
+2024-11-01 20:16:30,965 INFO    MainThread:874718 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-11-01 20:16:30,965 INFO    MainThread:874718 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-11-01 20:16:30,965 INFO    MainThread:874718 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-11-01 20:16:30,967 INFO    MainThread:874718 [wandb_init.py:init():911] run started, returning control to user process
+2024-11-01 20:16:30,967 INFO    MainThread:874718 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_nodeterministic', 'train_set': '10M', 'batch_size': 3, 'epoch': 6, 'seed': 0, 'lr': 5e-06}
+2024-11-01 20:16:30,974 WARNING MsgRouterThr:874718 [router.py:message_loop():77] message_loop has been closed

wandb/run-20241101_202058-hjyig8so/run-hjyig8so.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2bb18c91a178f700bc9b2e5d6d0e4bea78a095f878d6c225bc65c1b29e8d0dd1
+size 13297708

wandb/run-20241105_160652-v9udw9ab/files/config.yaml

@@ -0,0 +1,49 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.9.19
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "3":
+                - 13
+                - 23
+                - 55
+            "4": 3.9.19
+            "5": 0.18.5
+            "6": 4.45.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+batch_size:
+    value: 3
+epoch:
+    value: 3
+lr:
+    value: 5e-06
+perturbation:
+    value: shuffle_deterministic21
+seed:
+    value: 0
+train_set:
+    value: 10M

wandb/run-20241105_160652-v9udw9ab/files/output.log

@@ -0,0 +1,8 @@
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 165, in <module>
+    dataset = load_dataset('babylm_dataset_test.py', name=dataset_name, trust_remote_code=True)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/load.py", line 2096, in load_dataset
+    builder_instance.download_and_prepare(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/builder.py", line 875, in download_and_prepare
+    raise OSError(
+OSError: Not enough disk space. Needed: Unknown size (download: Unknown size, generated: Unknown size, post-processed: Unknown size)

wandb/run-20241105_160652-v9udw9ab/files/requirements.txt

@@ -0,0 +1,147 @@
+funcsigs==1.0.2
+sentry-sdk==2.17.0
+multiprocess==0.70.16
+numpy==1.26.2
+pluralizer==1.2.0
+debugpy==1.6.7
+nvidia-cudnn-cu11==8.5.0.96
+deepspeed==0.15.2
+data==0.4
+pandas==2.1.3
+tomli==2.0.1
+charset-normalizer==3.3.2
+attrs==24.2.0
+aiosignal==1.3.1
+fsspec==2023.10.0
+nvidia-cusparse-cu11==11.7.4.91
+zipp==3.12.0
+mypy-extensions==1.0.0
+datasets==3.0.1
+joblib==1.3.2
+hjson==3.1.0
+traitlets==5.7.1
+stack-data==0.6.0
+transformers==4.45.1
+sympy==1.11.1
+Pygments==2.15.0
+docker-pycreds==0.4.0
+dill==0.3.8
+wheel==0.44.0
+prompt-toolkit==3.0.30
+parso==0.8.3
+ipykernel==6.23.1
+pyarrow==17.0.0
+certifi==2023.11.17
+nvidia-cufft-cu11==10.9.0.58
+six==1.16.0
+pydantic==2.9.2
+click==8.1.7
+nest-asyncio==1.5.6
+gmpy2==2.1.0
+matplotlib==3.8.2
+scipy==1.11.4
+typing_extensions==4.12.2
+statsmodels==0.14.0
+huggingface-hub==0.25.0
+frozenlist==1.4.1
+gpustat==1.1.1
+nvidia-nvtx-cu11==11.7.91
+safetensors==0.4.5
+stanza==1.9.2
+decorator==5.1.1
+seaborn==0.13.0
+sentencepiece==0.2.0
+PyYAML==6.0.1
+black==24.8.0
+protobuf==4.25.1
+pickleshare==0.7.5
+peft==0.13.0
+triton==2.0.0
+nvidia-cuda-runtime-cu11==11.7.99
+Jinja2==3.1.2
+nvidia-cusolver-cu11==11.4.0.1
+executing==1.2.0
+jupyter_client==8.1.0
+pluggy==1.3.0
+cmake==3.30.3
+pytz==2023.3.post1
+aiohappyeyeballs==2.4.2
+kiwisolver==1.4.5
+py-cpuinfo==9.0.0
+Pillow==10.1.0
+ptyprocess==0.7.0
+importlib_resources==6.4.5
+GitPython==3.1.43
+importlib-metadata==6.0.0
+iniconfig==2.0.0
+scikit-learn==1.3.2
+exceptiongroup==1.1.0
+networkx==2.8.6
+accelerate==1.0.0
+nltk==3.8.1
+shutilwhich==1.1.0
+fonttools==4.45.1
+future==0.18.3
+aiohttp==3.10.6
+wcwidth==0.2.5
+idna==3.6
+filelock==3.12.2
+pathspec==0.12.1
+jupyter_core==5.1.0
+lit==18.1.8
+nvidia-curand-cu11==10.2.10.91
+nvidia-cublas-cu11==11.10.3.66
+nvidia-ml-py==12.560.30
+msgpack==1.1.0
+python-dateutil==2.8.2
+blessed==1.20.0
+packaging==23.0
+gitdb==4.0.11
+yarl==1.13.0
+emoji==2.8.0
+tzdata==2023.3
+cycler==0.12.1
+tornado==6.2
+backcall==0.2.0
+plotnine==0.12.4
+ninja==1.11.1.1
+latex==0.7.0
+wandb==0.18.5
+setproctitle==1.3.3
+threadpoolctl==3.2.0
+requests==2.32.3
+pyparsing==3.1.1
+smmap==5.0.1
+pyzmq==23.0.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+matplotlib-inline==0.1.6
+latexcodec==1.0.0
+ipython==8.0.0
+patsy==0.5.3
+contourpy==1.2.0
+multidict==6.1.0
+mizani==0.9.3
+urllib3==2.1.0
+tokenizers==0.20.0
+MarkupSafe==2.1.2
+pip==24.2
+pexpect==4.8.0
+tqdm==4.66.5
+jedi==0.18.2
+pydantic_core==2.23.4
+tempdir==0.7.1
+mpmath==1.2.1
+setuptools==72.1.0
+pytest==7.4.3
+pure-eval==0.2.2
+psutil==5.9.1
+comm==0.1.2
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+regex==2023.10.3
+platformdirs==2.5.2
+asttokens==2.2.1
+torch==2.0.0
+nvidia-nccl-cu11==2.14.3
+xxhash==3.5.0

wandb/run-20241105_160652-v9udw9ab/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-11-05T21:06:52.157764Z",
+  "args":  [
+    "--perturbation",
+    "shuffle_deterministic21",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "3",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1792542826496"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241105_160652-v9udw9ab/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":5}}

wandb/run-20241105_160652-v9udw9ab/logs/debug-internal.log

@@ -0,0 +1,17 @@
+{"time":"2024-11-05T16:06:52.159922257-05:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-11-05T16:06:52.159942657-05:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_160652-v9udw9ab/logs/debug-core.log"}
+{"time":"2024-11-05T16:06:52.268171392-05:00","level":"INFO","msg":"created new stream","id":"v9udw9ab"}
+{"time":"2024-11-05T16:06:52.268215922-05:00","level":"INFO","msg":"stream: started","id":"v9udw9ab"}
+{"time":"2024-11-05T16:06:52.268269323-05:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"v9udw9ab"}}
+{"time":"2024-11-05T16:06:52.268270183-05:00","level":"INFO","msg":"handler: started","stream_id":{"value":"v9udw9ab"}}
+{"time":"2024-11-05T16:06:52.268491884-05:00","level":"INFO","msg":"sender: started","stream_id":"v9udw9ab"}
+{"time":"2024-11-05T16:06:52.481746757-05:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-11-05T16:06:57.712126548-05:00","level":"INFO","msg":"stream: closing","id":"v9udw9ab"}
+{"time":"2024-11-05T16:06:57.712223249-05:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-11-05T16:06:57.713709807-05:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-11-05T16:06:57.799517789-05:00","level":"ERROR","msg":"sender: sendDefer: failed to build job artifact","error":"failed to write data to file: write /tmp/tmpfile-3332236271: no space left on device"}
+{"time":"2024-11-05T16:06:58.057468605-05:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-11-05T16:06:58.178861806-05:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"v9udw9ab"}}
+{"time":"2024-11-05T16:06:58.178915807-05:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"v9udw9ab"}}
+{"time":"2024-11-05T16:06:58.178935557-05:00","level":"INFO","msg":"sender: closed","stream_id":"v9udw9ab"}
+{"time":"2024-11-05T16:06:58.178979897-05:00","level":"INFO","msg":"stream: closed","id":"v9udw9ab"}

wandb/run-20241105_160652-v9udw9ab/logs/debug.log

@@ -0,0 +1,27 @@
+2024-11-05 16:06:52,154 INFO    MainThread:1771273 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-11-05 16:06:52,154 INFO    MainThread:1771273 [wandb_setup.py:_flush():79] Configure stats pid to 1771273
+2024-11-05 16:06:52,154 INFO    MainThread:1771273 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-11-05 16:06:52,154 INFO    MainThread:1771273 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-11-05 16:06:52,154 INFO    MainThread:1771273 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-11-05 16:06:52,154 INFO    MainThread:1771273 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-11-05 16:06:52,154 INFO    MainThread:1771273 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-11-05 16:06:52,154 INFO    MainThread:1771273 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-11-05 16:06:52,154 INFO    MainThread:1771273 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_160652-v9udw9ab/logs/debug.log
+2024-11-05 16:06:52,155 INFO    MainThread:1771273 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_160652-v9udw9ab/logs/debug-internal.log
+2024-11-05 16:06:52,155 INFO    MainThread:1771273 [wandb_init.py:init():621] calling init triggers
+2024-11-05 16:06:52,155 INFO    MainThread:1771273 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-11-05 16:06:52,155 INFO    MainThread:1771273 [wandb_init.py:init():671] starting backend
+2024-11-05 16:06:52,155 INFO    MainThread:1771273 [wandb_init.py:init():675] sending inform_init request
+2024-11-05 16:06:52,156 INFO    MainThread:1771273 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-11-05 16:06:52,157 INFO    MainThread:1771273 [wandb_init.py:init():688] backend started and connected
+2024-11-05 16:06:52,160 INFO    MainThread:1771273 [wandb_init.py:init():783] updated telemetry
+2024-11-05 16:06:52,189 INFO    MainThread:1771273 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-11-05 16:06:52,478 INFO    MainThread:1771273 [wandb_init.py:init():867] starting run threads in backend
+2024-11-05 16:06:52,566 INFO    MainThread:1771273 [wandb_run.py:_console_start():2463] atexit reg
+2024-11-05 16:06:52,566 INFO    MainThread:1771273 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-11-05 16:06:52,566 INFO    MainThread:1771273 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-11-05 16:06:52,566 INFO    MainThread:1771273 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-11-05 16:06:52,568 INFO    MainThread:1771273 [wandb_init.py:init():911] run started, returning control to user process
+2024-11-05 16:06:52,568 INFO    MainThread:1771273 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_deterministic21', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0, 'lr': 5e-06}
+2024-11-05 16:06:57,712 WARNING MsgRouterThr:1771273 [router.py:message_loop():77] message_loop has been closed

wandb/run-20241105_162858-hqnfirxi/files/config.yaml

@@ -0,0 +1,49 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.9.19
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "3":
+                - 13
+                - 23
+                - 55
+            "4": 3.9.19
+            "5": 0.18.5
+            "6": 4.45.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+batch_size:
+    value: 3
+epoch:
+    value: 3
+lr:
+    value: 5e-06
+perturbation:
+    value: shuffle_deterministic57
+seed:
+    value: 0
+train_set:
+    value: 10M