diff --git a/.gitattributes b/.gitattributes index 984c0f532f5b9629abb71dc949c00bde8d0184dd..a66db3e51f9ac2908391de9d50b408cd1edaa309 100644 --- a/.gitattributes +++ b/.gitattributes @@ -101,3 +101,5 @@ wandb/run-20241129_083813-1lx1o6j4/run-1lx1o6j4.wandb filter=lfs diff=lfs merge= wandb/run-20241101_200535-kh416n4m/run-kh416n4m.wandb filter=lfs diff=lfs merge=lfs -text wandb/run-20241101_202058-jijqbvs1/run-jijqbvs1.wandb filter=lfs diff=lfs merge=lfs -text wandb/run-20241129_083813-gsvlu1z8/run-gsvlu1z8.wandb filter=lfs diff=lfs merge=lfs -text +wandb/run-20241030_231835-o1t74f3e/run-o1t74f3e.wandb filter=lfs diff=lfs merge=lfs -text +wandb/run-20241030_233740-98qje3cr/run-98qje3cr.wandb filter=lfs diff=lfs merge=lfs -text diff --git a/wandb/run-20241030_010305-y2ohxj86/files/config.yaml b/wandb/run-20241030_010305-y2ohxj86/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f8546482f67cbb9041af9d756594322c485d67a9 --- /dev/null +++ b/wandb/run-20241030_010305-y2ohxj86/files/config.yaml @@ -0,0 +1,47 @@ +_wandb: + value: + cli_version: 0.18.5 + m: [] + python_version: 3.9.19 + t: + "1": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "2": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "3": + - 13 + - 23 + - 55 + "4": 3.9.19 + "5": 0.18.5 + "6": 4.45.1 + "8": + - 5 + "12": 0.18.5 + "13": linux-x86_64 +batch_size: + value: 3 +epoch: + value: 7 +perturbation: + value: reverse_control +seed: + value: 0 +train_set: + value: 10M diff --git a/wandb/run-20241030_010305-y2ohxj86/files/output.log b/wandb/run-20241030_010305-y2ohxj86/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..1777f063b107f651dddc063d1d3a3fa80015bf9e --- /dev/null +++ b/wandb/run-20241030_010305-y2ohxj86/files/output.log @@ -0,0 +1,4 @@ +Traceback (most recent call last): + File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 162, in + dataset_name = f"babylm_{args.perturbation}_{args.train_zset}_seed{args.seed}" +AttributeError: 'Namespace' object has no attribute 'train_zset' diff --git a/wandb/run-20241030_010305-y2ohxj86/files/wandb-metadata.json b/wandb/run-20241030_010305-y2ohxj86/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..550970f4f8c0f27ad7d1393e3bc081b3050f6ccc --- /dev/null +++ b/wandb/run-20241030_010305-y2ohxj86/files/wandb-metadata.json @@ -0,0 +1,97 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-10-30T05:03:05.880128Z", + "args": [ + "--perturbation", + "reverse_control", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "7", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", + "codePath": "train/train_deep_wandb.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_deep_wandb.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA RTX A6000", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888559353856", + "used": "1719200043008" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + } + ], + "cudaVersion": "11.8" +} \ No newline at end of file diff --git a/wandb/run-20241030_010305-y2ohxj86/files/wandb-summary.json b/wandb/run-20241030_010305-y2ohxj86/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..6c37fe1cbbb8aed86fd461a79642cb991e4d35cf --- /dev/null +++ b/wandb/run-20241030_010305-y2ohxj86/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":0}} \ No newline at end of file diff --git a/wandb/run-20241030_010305-y2ohxj86/logs/debug.log b/wandb/run-20241030_010305-y2ohxj86/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..526361526242926722ed2d88e9eb4a6e2fa7dc62 --- /dev/null +++ b/wandb/run-20241030_010305-y2ohxj86/logs/debug.log @@ -0,0 +1,27 @@ +2024-10-30 01:03:05,878 INFO MainThread:320660 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-10-30 01:03:05,878 INFO MainThread:320660 [wandb_setup.py:_flush():79] Configure stats pid to 320660 +2024-10-30 01:03:05,878 INFO MainThread:320660 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-10-30 01:03:05,878 INFO MainThread:320660 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-10-30 01:03:05,878 INFO MainThread:320660 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-10-30 01:03:05,878 INFO MainThread:320660 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-10-30 01:03:05,878 INFO MainThread:320660 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'} +2024-10-30 01:03:05,878 INFO MainThread:320660 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-10-30 01:03:05,878 INFO MainThread:320660 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_010305-y2ohxj86/logs/debug.log +2024-10-30 01:03:05,878 INFO MainThread:320660 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_010305-y2ohxj86/logs/debug-internal.log +2024-10-30 01:03:05,878 INFO MainThread:320660 [wandb_init.py:init():621] calling init triggers +2024-10-30 01:03:05,878 INFO MainThread:320660 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-10-30 01:03:05,878 INFO MainThread:320660 [wandb_init.py:init():671] starting backend +2024-10-30 01:03:05,878 INFO MainThread:320660 [wandb_init.py:init():675] sending inform_init request +2024-10-30 01:03:05,879 INFO MainThread:320660 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-10-30 01:03:05,879 INFO MainThread:320660 [wandb_init.py:init():688] backend started and connected +2024-10-30 01:03:05,883 INFO MainThread:320660 [wandb_init.py:init():783] updated telemetry +2024-10-30 01:03:05,914 INFO MainThread:320660 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-10-30 01:03:06,150 INFO MainThread:320660 [wandb_init.py:init():867] starting run threads in backend +2024-10-30 01:03:06,258 INFO MainThread:320660 [wandb_run.py:_console_start():2463] atexit reg +2024-10-30 01:03:06,258 INFO MainThread:320660 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-10-30 01:03:06,258 INFO MainThread:320660 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-10-30 01:03:06,258 INFO MainThread:320660 [wandb_run.py:_redirect():2401] Redirects installed. +2024-10-30 01:03:06,260 INFO MainThread:320660 [wandb_init.py:init():911] run started, returning control to user process +2024-10-30 01:03:06,260 INFO MainThread:320660 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_control', 'train_set': '10M', 'batch_size': 3, 'epoch': 7, 'seed': 0} +2024-10-30 01:03:06,261 WARNING MsgRouterThr:320660 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20241030_010305-y2ohxj86/run-y2ohxj86.wandb b/wandb/run-20241030_010305-y2ohxj86/run-y2ohxj86.wandb new file mode 100644 index 0000000000000000000000000000000000000000..72d57823fdc814d00dc3a643119cd06053cd88e5 Binary files /dev/null and b/wandb/run-20241030_010305-y2ohxj86/run-y2ohxj86.wandb differ diff --git a/wandb/run-20241030_013141-ie4x9mdj/files/config.yaml b/wandb/run-20241030_013141-ie4x9mdj/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d471293c627dbaeaf5be47c709016b3dfbf28c6c --- /dev/null +++ b/wandb/run-20241030_013141-ie4x9mdj/files/config.yaml @@ -0,0 +1,47 @@ +_wandb: + value: + cli_version: 0.18.5 + m: [] + python_version: 3.9.19 + t: + "1": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "2": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "3": + - 13 + - 23 + - 55 + "4": 3.9.19 + "5": 0.18.5 + "6": 4.45.1 + "8": + - 5 + "12": 0.18.5 + "13": linux-x86_64 +batch_size: + value: 3 +epoch: + value: 7 +perturbation: + value: reverse_full +seed: + value: 0 +train_set: + value: 10M diff --git a/wandb/run-20241030_013141-ie4x9mdj/files/output.log b/wandb/run-20241030_013141-ie4x9mdj/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..17990ba13db7c58eaf274a03a6e6e8a94645fcf4 --- /dev/null +++ b/wandb/run-20241030_013141-ie4x9mdj/files/output.log @@ -0,0 +1,35 @@ +config.json: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 844/844 [00:00<00:00, 416kB/s] +Downloading shards: 0%| | 0/2 [01:04 + model = AutoModelForCausalLM.from_pretrained(model_name, + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/models/auto/auto_factory.py", line 564, in from_pretrained + return model_class.from_pretrained( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/modeling_utils.py", line 3769, in from_pretrained + resolved_archive_file, sharded_metadata = get_checkpoint_shard_files( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/utils/hub.py", line 1098, in get_checkpoint_shard_files + cached_filename = cached_file( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/utils/hub.py", line 403, in cached_file + resolved_file = hf_hub_download( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/utils/_deprecation.py", line 101, in inner_f + return f(*args, **kwargs) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn + return fn(*args, **kwargs) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1232, in hf_hub_download + return _hf_hub_download_to_cache_dir( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1380, in _hf_hub_download_to_cache_dir + with WeakFileLock(lock_path): + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/contextlib.py", line 119, in __enter__ + return next(self.gen) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/utils/_fixes.py", line 98, in WeakFileLock + lock.acquire() + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/filelock/_api.py", line 225, in acquire + time.sleep(poll_interval) +KeyboardInterrupt diff --git a/wandb/run-20241030_013141-ie4x9mdj/files/requirements.txt b/wandb/run-20241030_013141-ie4x9mdj/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..95a931302e269cc9e4fa5b719b6511f176ee2416 --- /dev/null +++ b/wandb/run-20241030_013141-ie4x9mdj/files/requirements.txt @@ -0,0 +1,147 @@ +funcsigs==1.0.2 +sentry-sdk==2.17.0 +multiprocess==0.70.16 +numpy==1.26.2 +pluralizer==1.2.0 +debugpy==1.6.7 +nvidia-cudnn-cu11==8.5.0.96 +deepspeed==0.15.2 +data==0.4 +pandas==2.1.3 +tomli==2.0.1 +charset-normalizer==3.3.2 +attrs==24.2.0 +aiosignal==1.3.1 +fsspec==2023.10.0 +nvidia-cusparse-cu11==11.7.4.91 +zipp==3.12.0 +mypy-extensions==1.0.0 +datasets==3.0.1 +joblib==1.3.2 +hjson==3.1.0 +traitlets==5.7.1 +stack-data==0.6.0 +transformers==4.45.1 +sympy==1.11.1 +Pygments==2.15.0 +docker-pycreds==0.4.0 +dill==0.3.8 +wheel==0.44.0 +prompt-toolkit==3.0.30 +parso==0.8.3 +ipykernel==6.23.1 +pyarrow==17.0.0 +certifi==2023.11.17 +nvidia-cufft-cu11==10.9.0.58 +six==1.16.0 +pydantic==2.9.2 +click==8.1.7 +nest-asyncio==1.5.6 +gmpy2==2.1.0 +matplotlib==3.8.2 +scipy==1.11.4 +typing_extensions==4.12.2 +statsmodels==0.14.0 +huggingface-hub==0.25.0 +frozenlist==1.4.1 +gpustat==1.1.1 +nvidia-nvtx-cu11==11.7.91 +safetensors==0.4.5 +stanza==1.9.2 +decorator==5.1.1 +seaborn==0.13.0 +sentencepiece==0.2.0 +PyYAML==6.0.1 +black==24.8.0 +protobuf==4.25.1 +pickleshare==0.7.5 +peft==0.13.0 +triton==2.0.0 +nvidia-cuda-runtime-cu11==11.7.99 +Jinja2==3.1.2 +nvidia-cusolver-cu11==11.4.0.1 +executing==1.2.0 +jupyter_client==8.1.0 +pluggy==1.3.0 +cmake==3.30.3 +pytz==2023.3.post1 +aiohappyeyeballs==2.4.2 +kiwisolver==1.4.5 +py-cpuinfo==9.0.0 +Pillow==10.1.0 +ptyprocess==0.7.0 +importlib_resources==6.4.5 +GitPython==3.1.43 +importlib-metadata==6.0.0 +iniconfig==2.0.0 +scikit-learn==1.3.2 +exceptiongroup==1.1.0 +networkx==2.8.6 +accelerate==1.0.0 +nltk==3.8.1 +shutilwhich==1.1.0 +fonttools==4.45.1 +future==0.18.3 +aiohttp==3.10.6 +wcwidth==0.2.5 +idna==3.6 +filelock==3.12.2 +pathspec==0.12.1 +jupyter_core==5.1.0 +lit==18.1.8 +nvidia-curand-cu11==10.2.10.91 +nvidia-cublas-cu11==11.10.3.66 +nvidia-ml-py==12.560.30 +msgpack==1.1.0 +python-dateutil==2.8.2 +blessed==1.20.0 +packaging==23.0 +gitdb==4.0.11 +yarl==1.13.0 +emoji==2.8.0 +tzdata==2023.3 +cycler==0.12.1 +tornado==6.2 +backcall==0.2.0 +plotnine==0.12.4 +ninja==1.11.1.1 +latex==0.7.0 +wandb==0.18.5 +setproctitle==1.3.3 +threadpoolctl==3.2.0 +requests==2.32.3 +pyparsing==3.1.1 +smmap==5.0.1 +pyzmq==23.0.0 +async-timeout==4.0.3 +annotated-types==0.7.0 +matplotlib-inline==0.1.6 +latexcodec==1.0.0 +ipython==8.0.0 +patsy==0.5.3 +contourpy==1.2.0 +multidict==6.1.0 +mizani==0.9.3 +urllib3==2.1.0 +tokenizers==0.20.0 +MarkupSafe==2.1.2 +pip==24.2 +pexpect==4.8.0 +tqdm==4.66.5 +jedi==0.18.2 +pydantic_core==2.23.4 +tempdir==0.7.1 +mpmath==1.2.1 +setuptools==72.1.0 +pytest==7.4.3 +pure-eval==0.2.2 +psutil==5.9.1 +comm==0.1.2 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +regex==2023.10.3 +platformdirs==2.5.2 +asttokens==2.2.1 +torch==2.0.0 +nvidia-nccl-cu11==2.14.3 +xxhash==3.5.0 diff --git a/wandb/run-20241030_013141-ie4x9mdj/files/wandb-metadata.json b/wandb/run-20241030_013141-ie4x9mdj/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..13551a9309419b1a9c862e1bea422200873da751 --- /dev/null +++ b/wandb/run-20241030_013141-ie4x9mdj/files/wandb-metadata.json @@ -0,0 +1,97 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-10-30T05:31:41.778763Z", + "args": [ + "--perturbation", + "reverse_full", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "7", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", + "codePath": "train/train_deep_wandb.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_deep_wandb.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA RTX A6000", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888559353856", + "used": "1709824413696" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + } + ], + "cudaVersion": "11.8" +} \ No newline at end of file diff --git a/wandb/run-20241030_013141-ie4x9mdj/files/wandb-summary.json b/wandb/run-20241030_013141-ie4x9mdj/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..b155621703e444311fe2da9b782a2b70b5491169 --- /dev/null +++ b/wandb/run-20241030_013141-ie4x9mdj/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":94}} \ No newline at end of file diff --git a/wandb/run-20241030_013141-ie4x9mdj/logs/debug-internal.log b/wandb/run-20241030_013141-ie4x9mdj/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..10a13fc960c69e56c9dee9b3fd926c14f0520ea7 --- /dev/null +++ b/wandb/run-20241030_013141-ie4x9mdj/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2024-10-30T01:31:41.781725348-04:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-10-30T01:31:41.781739018-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_013141-ie4x9mdj/logs/debug-core.log"} +{"time":"2024-10-30T01:31:41.891449316-04:00","level":"INFO","msg":"created new stream","id":"ie4x9mdj"} +{"time":"2024-10-30T01:31:41.891511467-04:00","level":"INFO","msg":"stream: started","id":"ie4x9mdj"} +{"time":"2024-10-30T01:31:41.891546597-04:00","level":"INFO","msg":"sender: started","stream_id":"ie4x9mdj"} +{"time":"2024-10-30T01:31:41.891536947-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"ie4x9mdj"}} +{"time":"2024-10-30T01:31:41.891563967-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"ie4x9mdj"}} +{"time":"2024-10-30T01:31:42.085980534-04:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-10-30T01:33:16.59613712-04:00","level":"INFO","msg":"stream: closing","id":"ie4x9mdj"} +{"time":"2024-10-30T01:33:16.59622584-04:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2024-10-30T01:33:16.597521969-04:00","level":"INFO","msg":"Stopped system monitor"} diff --git a/wandb/run-20241030_013141-ie4x9mdj/logs/debug.log b/wandb/run-20241030_013141-ie4x9mdj/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..62ce0322501f4c92d6c3499082dbbc42a4cea73c --- /dev/null +++ b/wandb/run-20241030_013141-ie4x9mdj/logs/debug.log @@ -0,0 +1,27 @@ +2024-10-30 01:31:41,775 INFO MainThread:335758 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-10-30 01:31:41,775 INFO MainThread:335758 [wandb_setup.py:_flush():79] Configure stats pid to 335758 +2024-10-30 01:31:41,775 INFO MainThread:335758 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-10-30 01:31:41,775 INFO MainThread:335758 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-10-30 01:31:41,775 INFO MainThread:335758 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-10-30 01:31:41,775 INFO MainThread:335758 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-10-30 01:31:41,776 INFO MainThread:335758 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'} +2024-10-30 01:31:41,776 INFO MainThread:335758 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-10-30 01:31:41,776 INFO MainThread:335758 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_013141-ie4x9mdj/logs/debug.log +2024-10-30 01:31:41,776 INFO MainThread:335758 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_013141-ie4x9mdj/logs/debug-internal.log +2024-10-30 01:31:41,776 INFO MainThread:335758 [wandb_init.py:init():621] calling init triggers +2024-10-30 01:31:41,776 INFO MainThread:335758 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-10-30 01:31:41,776 INFO MainThread:335758 [wandb_init.py:init():671] starting backend +2024-10-30 01:31:41,776 INFO MainThread:335758 [wandb_init.py:init():675] sending inform_init request +2024-10-30 01:31:41,777 INFO MainThread:335758 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-10-30 01:31:41,778 INFO MainThread:335758 [wandb_init.py:init():688] backend started and connected +2024-10-30 01:31:41,781 INFO MainThread:335758 [wandb_init.py:init():783] updated telemetry +2024-10-30 01:31:41,811 INFO MainThread:335758 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-10-30 01:31:42,082 INFO MainThread:335758 [wandb_init.py:init():867] starting run threads in backend +2024-10-30 01:31:42,173 INFO MainThread:335758 [wandb_run.py:_console_start():2463] atexit reg +2024-10-30 01:31:42,173 INFO MainThread:335758 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-10-30 01:31:42,174 INFO MainThread:335758 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-10-30 01:31:42,174 INFO MainThread:335758 [wandb_run.py:_redirect():2401] Redirects installed. +2024-10-30 01:31:42,175 INFO MainThread:335758 [wandb_init.py:init():911] run started, returning control to user process +2024-10-30 01:31:42,175 INFO MainThread:335758 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_full', 'train_set': '10M', 'batch_size': 3, 'epoch': 7, 'seed': 0} +2024-10-30 01:33:16,596 WARNING MsgRouterThr:335758 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20241030_013141-ie4x9mdj/run-ie4x9mdj.wandb b/wandb/run-20241030_013141-ie4x9mdj/run-ie4x9mdj.wandb new file mode 100644 index 0000000000000000000000000000000000000000..e8f623a17dfa52dbea7b103a40cef0abba4d6022 Binary files /dev/null and b/wandb/run-20241030_013141-ie4x9mdj/run-ie4x9mdj.wandb differ diff --git a/wandb/run-20241030_231835-o1t74f3e/run-o1t74f3e.wandb b/wandb/run-20241030_231835-o1t74f3e/run-o1t74f3e.wandb new file mode 100644 index 0000000000000000000000000000000000000000..4f8b6970bcbd84bd5cb9a0ba500cf0bbbc8a6b89 --- /dev/null +++ b/wandb/run-20241030_231835-o1t74f3e/run-o1t74f3e.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38ce013a7d495d13853ea95c875ac41642d801349e6a601213f10ccf7e0a2284 +size 655360 diff --git a/wandb/run-20241030_233740-98qje3cr/run-98qje3cr.wandb b/wandb/run-20241030_233740-98qje3cr/run-98qje3cr.wandb new file mode 100644 index 0000000000000000000000000000000000000000..01ad28647f53fe4ea97f0c396cdbfa03653d8083 --- /dev/null +++ b/wandb/run-20241030_233740-98qje3cr/run-98qje3cr.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57002d575e8101f28e83eadd110e5e246e26a1c57b86b0a296f93e3044577c09 +size 851968 diff --git a/wandb/run-20241031_002020-t2atf7xf/files/config.yaml b/wandb/run-20241031_002020-t2atf7xf/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3b75df220307eb3060a9657bb8b75f0ac42fdd90 --- /dev/null +++ b/wandb/run-20241031_002020-t2atf7xf/files/config.yaml @@ -0,0 +1,50 @@ +_wandb: + value: + cli_version: 0.18.5 + m: [] + python_version: 3.9.19 + t: + "1": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "2": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "3": + - 2 + - 13 + - 23 + - 55 + "4": 3.9.19 + "5": 0.18.5 + "6": 4.45.1 + "8": + - 5 + "12": 0.18.5 + "13": linux-x86_64 +batch_size: + value: 3 +epoch: + value: 6 +lr: + value: 1e-05 +perturbation: + value: reverse_full +seed: + value: 0 +train_set: + value: 10M diff --git a/wandb/run-20241031_002020-t2atf7xf/files/output.log b/wandb/run-20241031_002020-t2atf7xf/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..1fb4a1bc9dde93ae562da554472388aa392c31cd --- /dev/null +++ b/wandb/run-20241031_002020-t2atf7xf/files/output.log @@ -0,0 +1,14 @@ +Downloading shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [02:33<00:00, 76.70s/it] +Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:05<00:00, 2.86s/it] +tokenized_valid: Dataset({ + features: ['input_ids', 'attention_mask'], + num_rows: 600 +}) +/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +[2024-10-31 00:23:02,660] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-10-31 00:23:12,358] [INFO] [comm.py:652:init_distributed] cdb=None +Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination +Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root... +Loading extension module cpu_adam... +Time to load cpu_adam op: 5.453424692153931 seconds diff --git a/wandb/run-20241031_002020-t2atf7xf/files/wandb-metadata.json b/wandb/run-20241031_002020-t2atf7xf/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c62cce1050f73b0697097884672926c037bb83b5 --- /dev/null +++ b/wandb/run-20241031_002020-t2atf7xf/files/wandb-metadata.json @@ -0,0 +1,97 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-10-31T04:20:20.142252Z", + "args": [ + "--perturbation", + "reverse_full", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "6", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", + "codePath": "train/train_deep_wandb.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_deep_wandb.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA RTX A6000", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888559353856", + "used": "1735153242112" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + } + ], + "cudaVersion": "11.8" +} \ No newline at end of file diff --git a/wandb/run-20241031_002020-t2atf7xf/files/wandb-summary.json b/wandb/run-20241031_002020-t2atf7xf/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..4f8a2d8f302b1f64c484c31f4742ad6275018b3d --- /dev/null +++ b/wandb/run-20241031_002020-t2atf7xf/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":32219}} \ No newline at end of file diff --git a/wandb/run-20241031_002020-t2atf7xf/logs/debug-internal.log b/wandb/run-20241031_002020-t2atf7xf/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..4bc7885eb2ef791b5fd76d76a61d21bcf100aec9 --- /dev/null +++ b/wandb/run-20241031_002020-t2atf7xf/logs/debug-internal.log @@ -0,0 +1,17 @@ +{"time":"2024-10-31T00:20:20.144856997-04:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-10-31T00:20:20.144879858-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_002020-t2atf7xf/logs/debug-core.log"} +{"time":"2024-10-31T00:20:20.252562662-04:00","level":"INFO","msg":"created new stream","id":"t2atf7xf"} +{"time":"2024-10-31T00:20:20.253136626-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"t2atf7xf"}} +{"time":"2024-10-31T00:20:20.253250477-04:00","level":"INFO","msg":"sender: started","stream_id":"t2atf7xf"} +{"time":"2024-10-31T00:20:20.253229647-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"t2atf7xf"}} +{"time":"2024-10-31T00:20:20.253218787-04:00","level":"INFO","msg":"stream: started","id":"t2atf7xf"} +{"time":"2024-10-31T00:20:21.434842431-04:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-10-31T09:17:19.826967528-04:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2024-10-31T09:17:19.903947447-04:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2024-10-31T09:17:20.764796815-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"saving job artifact","runtime_seconds":0.608821579}],"total_operations":1}} +{"time":"2024-10-31T09:17:21.827392577-04:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2024-10-31T09:17:22.951276585-04:00","level":"INFO","msg":"stream: closing","id":"t2atf7xf"} +{"time":"2024-10-31T09:17:22.951306535-04:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"t2atf7xf"}} +{"time":"2024-10-31T09:17:22.951341595-04:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"t2atf7xf"}} +{"time":"2024-10-31T09:17:22.951367016-04:00","level":"INFO","msg":"sender: closed","stream_id":"t2atf7xf"} +{"time":"2024-10-31T09:17:22.951451446-04:00","level":"INFO","msg":"stream: closed","id":"t2atf7xf"} diff --git a/wandb/run-20241031_002020-t2atf7xf/logs/debug.log b/wandb/run-20241031_002020-t2atf7xf/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..496d733ec24c230717492c137dc94b31fb1e5746 --- /dev/null +++ b/wandb/run-20241031_002020-t2atf7xf/logs/debug.log @@ -0,0 +1,33 @@ +2024-10-31 00:20:20,139 INFO MainThread:484457 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-10-31 00:20:20,139 INFO MainThread:484457 [wandb_setup.py:_flush():79] Configure stats pid to 484457 +2024-10-31 00:20:20,139 INFO MainThread:484457 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-10-31 00:20:20,139 INFO MainThread:484457 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-10-31 00:20:20,139 INFO MainThread:484457 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-10-31 00:20:20,139 INFO MainThread:484457 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-10-31 00:20:20,139 INFO MainThread:484457 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'} +2024-10-31 00:20:20,139 INFO MainThread:484457 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-10-31 00:20:20,139 INFO MainThread:484457 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_002020-t2atf7xf/logs/debug.log +2024-10-31 00:20:20,139 INFO MainThread:484457 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_002020-t2atf7xf/logs/debug-internal.log +2024-10-31 00:20:20,140 INFO MainThread:484457 [wandb_init.py:init():621] calling init triggers +2024-10-31 00:20:20,140 INFO MainThread:484457 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-10-31 00:20:20,140 INFO MainThread:484457 [wandb_init.py:init():671] starting backend +2024-10-31 00:20:20,140 INFO MainThread:484457 [wandb_init.py:init():675] sending inform_init request +2024-10-31 00:20:20,141 INFO MainThread:484457 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-10-31 00:20:20,141 INFO MainThread:484457 [wandb_init.py:init():688] backend started and connected +2024-10-31 00:20:20,147 INFO MainThread:484457 [wandb_init.py:init():783] updated telemetry +2024-10-31 00:20:20,170 INFO MainThread:484457 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-10-31 00:20:21,431 INFO MainThread:484457 [wandb_init.py:init():867] starting run threads in backend +2024-10-31 00:20:21,530 INFO MainThread:484457 [wandb_run.py:_console_start():2463] atexit reg +2024-10-31 00:20:21,530 INFO MainThread:484457 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-10-31 00:20:21,530 INFO MainThread:484457 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-10-31 00:20:21,530 INFO MainThread:484457 [wandb_run.py:_redirect():2401] Redirects installed. +2024-10-31 00:20:21,532 INFO MainThread:484457 [wandb_init.py:init():911] run started, returning control to user process +2024-10-31 00:20:21,532 INFO MainThread:484457 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_full', 'train_set': '10M', 'batch_size': 3, 'epoch': 6, 'seed': 0, 'lr': 1e-05} +2024-10-31 09:17:19,684 INFO MainThread:484457 [wandb_run.py:_finish():2158] finishing run yaning1001-dartmouth-college/impossible_llm_reverse/t2atf7xf +2024-10-31 09:17:19,700 INFO MainThread:484457 [wandb_run.py:_atexit_cleanup():2426] got exitcode: 0 +2024-10-31 09:17:19,748 INFO MainThread:484457 [wandb_run.py:_restore():2408] restore +2024-10-31 09:17:19,749 INFO MainThread:484457 [wandb_run.py:_restore():2414] restore done +2024-10-31 09:17:22,944 INFO MainThread:484457 [wandb_run.py:_footer_history_summary_info():3975] rendering history +2024-10-31 09:17:22,944 INFO MainThread:484457 [wandb_run.py:_footer_history_summary_info():4007] rendering summary +2024-10-31 09:17:22,950 INFO MainThread:484457 [wandb_run.py:_footer_sync_info():3934] logging synced files diff --git a/wandb/run-20241101_012612-71qkhmiu/files/config.yaml b/wandb/run-20241101_012612-71qkhmiu/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16d25ee0e4092474360045faaf0959cfc1b3e91d --- /dev/null +++ b/wandb/run-20241101_012612-71qkhmiu/files/config.yaml @@ -0,0 +1,49 @@ +_wandb: + value: + cli_version: 0.18.5 + m: [] + python_version: 3.9.19 + t: + "1": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "2": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "3": + - 13 + - 23 + - 55 + "4": 3.9.19 + "5": 0.18.5 + "6": 4.45.1 + "8": + - 5 + "12": 0.18.5 + "13": linux-x86_64 +batch_size: + value: 3 +epoch: + value: 6 +lr: + value: 5e-06 +perturbation: + value: shuffle_nodeterministic +seed: + value: 0 +train_set: + value: 10M diff --git a/wandb/run-20241101_012612-71qkhmiu/files/output.log b/wandb/run-20241101_012612-71qkhmiu/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..5824b3c1b0e7410256b3374a200db5bd3cc11c9d --- /dev/null +++ b/wandb/run-20241101_012612-71qkhmiu/files/output.log @@ -0,0 +1,12 @@ +Traceback (most recent call last): + File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 164, in + dataset = load_dataset('babylm_dataset_test.py', name=dataset_name, trust_remote_code=True) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/load.py", line 2074, in load_dataset + builder_instance = load_dataset_builder( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/load.py", line 1832, in load_dataset_builder + builder_instance: DatasetBuilder = builder_cls( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/builder.py", line 342, in __init__ + self.config, self.config_id = self._create_builder_config( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/builder.py", line 569, in _create_builder_config + raise ValueError( +ValueError: BuilderConfig 'babylm_shuffle_nodeterministic_10M_seed0' not found. Available: ['babylm_hop_control_10M_seed0', 'babylm_hop_tokens4_10M_seed0', 'babylm_hop_words4_10M_seed0', 'babylm_reverse_control_10M_seed0', 'babylm_reverse_partial_10M_seed0', 'babylm_reverse_full_10M_seed0', 'babylm_shuffle_control_10M_seed0', 'babylm_shuffle_nondeterministic_10M_seed0', 'babylm_shuffle_deterministic21_10M_seed0', 'babylm_shuffle_deterministic57_10M_seed0', 'babylm_shuffle_deterministic84_10M_seed0', 'babylm_shuffle_local3_10M_seed0', 'babylm_shuffle_local5_10M_seed0', 'babylm_shuffle_local10_10M_seed0', 'babylm_shuffle_even_odd_10M_seed0'] diff --git a/wandb/run-20241101_012612-71qkhmiu/files/wandb-metadata.json b/wandb/run-20241101_012612-71qkhmiu/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..e95db82e4cd709d102cd6f2c0fc3c1344cf4bffe --- /dev/null +++ b/wandb/run-20241101_012612-71qkhmiu/files/wandb-metadata.json @@ -0,0 +1,97 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-11-01T05:26:12.633153Z", + "args": [ + "--perturbation", + "shuffle_nodeterministic", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "6", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", + "codePath": "train/train_deep_wandb.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_deep_wandb.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA RTX A6000", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888559353856", + "used": "1753992224768" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + } + ], + "cudaVersion": "11.8" +} \ No newline at end of file diff --git a/wandb/run-20241101_012612-71qkhmiu/files/wandb-summary.json b/wandb/run-20241101_012612-71qkhmiu/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..6c37fe1cbbb8aed86fd461a79642cb991e4d35cf --- /dev/null +++ b/wandb/run-20241101_012612-71qkhmiu/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":0}} \ No newline at end of file diff --git a/wandb/run-20241101_012612-71qkhmiu/logs/debug-internal.log b/wandb/run-20241101_012612-71qkhmiu/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..0a0cbc365ebf2a7890518f904037a644fde43fb7 --- /dev/null +++ b/wandb/run-20241101_012612-71qkhmiu/logs/debug-internal.log @@ -0,0 +1,16 @@ +{"time":"2024-11-01T01:26:12.636802094-04:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-11-01T01:26:12.636827225-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_012612-71qkhmiu/logs/debug-core.log"} +{"time":"2024-11-01T01:26:12.746392665-04:00","level":"INFO","msg":"created new stream","id":"71qkhmiu"} +{"time":"2024-11-01T01:26:12.746437585-04:00","level":"INFO","msg":"stream: started","id":"71qkhmiu"} +{"time":"2024-11-01T01:26:12.746483116-04:00","level":"INFO","msg":"sender: started","stream_id":"71qkhmiu"} +{"time":"2024-11-01T01:26:12.746489706-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"71qkhmiu"}} +{"time":"2024-11-01T01:26:12.746479956-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"71qkhmiu"}} +{"time":"2024-11-01T01:26:12.954567681-04:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-11-01T01:26:13.065163989-04:00","level":"INFO","msg":"stream: closing","id":"71qkhmiu"} +{"time":"2024-11-01T01:26:13.065199569-04:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2024-11-01T01:26:13.065819814-04:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2024-11-01T01:26:13.640887088-04:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2024-11-01T01:26:13.772963695-04:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"71qkhmiu"}} +{"time":"2024-11-01T01:26:13.773103686-04:00","level":"INFO","msg":"sender: closed","stream_id":"71qkhmiu"} +{"time":"2024-11-01T01:26:13.773082126-04:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"71qkhmiu"}} +{"time":"2024-11-01T01:26:13.773279647-04:00","level":"INFO","msg":"stream: closed","id":"71qkhmiu"} diff --git a/wandb/run-20241101_012612-71qkhmiu/logs/debug.log b/wandb/run-20241101_012612-71qkhmiu/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..33c640247e8241cc5819a54fc8ad4dfdb7d05a35 --- /dev/null +++ b/wandb/run-20241101_012612-71qkhmiu/logs/debug.log @@ -0,0 +1,27 @@ +2024-11-01 01:26:12,631 INFO MainThread:677633 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-11-01 01:26:12,631 INFO MainThread:677633 [wandb_setup.py:_flush():79] Configure stats pid to 677633 +2024-11-01 01:26:12,631 INFO MainThread:677633 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-11-01 01:26:12,631 INFO MainThread:677633 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-11-01 01:26:12,631 INFO MainThread:677633 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-01 01:26:12,631 INFO MainThread:677633 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-01 01:26:12,631 INFO MainThread:677633 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'} +2024-11-01 01:26:12,631 INFO MainThread:677633 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-01 01:26:12,631 INFO MainThread:677633 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_012612-71qkhmiu/logs/debug.log +2024-11-01 01:26:12,631 INFO MainThread:677633 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_012612-71qkhmiu/logs/debug-internal.log +2024-11-01 01:26:12,631 INFO MainThread:677633 [wandb_init.py:init():621] calling init triggers +2024-11-01 01:26:12,631 INFO MainThread:677633 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-11-01 01:26:12,631 INFO MainThread:677633 [wandb_init.py:init():671] starting backend +2024-11-01 01:26:12,631 INFO MainThread:677633 [wandb_init.py:init():675] sending inform_init request +2024-11-01 01:26:12,632 INFO MainThread:677633 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-01 01:26:12,633 INFO MainThread:677633 [wandb_init.py:init():688] backend started and connected +2024-11-01 01:26:12,635 INFO MainThread:677633 [wandb_init.py:init():783] updated telemetry +2024-11-01 01:26:12,659 INFO MainThread:677633 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-11-01 01:26:12,949 INFO MainThread:677633 [wandb_init.py:init():867] starting run threads in backend +2024-11-01 01:26:13,053 INFO MainThread:677633 [wandb_run.py:_console_start():2463] atexit reg +2024-11-01 01:26:13,053 INFO MainThread:677633 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-11-01 01:26:13,054 INFO MainThread:677633 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-11-01 01:26:13,054 INFO MainThread:677633 [wandb_run.py:_redirect():2401] Redirects installed. +2024-11-01 01:26:13,055 INFO MainThread:677633 [wandb_init.py:init():911] run started, returning control to user process +2024-11-01 01:26:13,055 INFO MainThread:677633 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_nodeterministic', 'train_set': '10M', 'batch_size': 3, 'epoch': 6, 'seed': 0, 'lr': 5e-06} +2024-11-01 01:26:13,065 WARNING MsgRouterThr:677633 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20241101_012612-71qkhmiu/run-71qkhmiu.wandb b/wandb/run-20241101_012612-71qkhmiu/run-71qkhmiu.wandb new file mode 100644 index 0000000000000000000000000000000000000000..680c840d850775e59136b58d8e33d4f181f560cb Binary files /dev/null and b/wandb/run-20241101_012612-71qkhmiu/run-71qkhmiu.wandb differ diff --git a/wandb/run-20241101_093116-a95c3tmd/files/output.log b/wandb/run-20241101_093116-a95c3tmd/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..118d38dc3fcd8e77bd3d16407bc25062284c0c1b --- /dev/null +++ b/wandb/run-20241101_093116-a95c3tmd/files/output.log @@ -0,0 +1,13 @@ +Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:18<00:00, 9.07s/it] +tokenized_valid: Dataset({ + features: ['input_ids', 'attention_mask'], + num_rows: 600 +}) +/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +[2024-11-01 09:31:36,949] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-11-01 09:31:44,876] [INFO] [comm.py:652:init_distributed] cdb=None +Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination +Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root... +Loading extension module cpu_adam... +Time to load cpu_adam op: 4.957409381866455 seconds diff --git a/wandb/run-20241101_093116-a95c3tmd/files/requirements.txt b/wandb/run-20241101_093116-a95c3tmd/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..95a931302e269cc9e4fa5b719b6511f176ee2416 --- /dev/null +++ b/wandb/run-20241101_093116-a95c3tmd/files/requirements.txt @@ -0,0 +1,147 @@ +funcsigs==1.0.2 +sentry-sdk==2.17.0 +multiprocess==0.70.16 +numpy==1.26.2 +pluralizer==1.2.0 +debugpy==1.6.7 +nvidia-cudnn-cu11==8.5.0.96 +deepspeed==0.15.2 +data==0.4 +pandas==2.1.3 +tomli==2.0.1 +charset-normalizer==3.3.2 +attrs==24.2.0 +aiosignal==1.3.1 +fsspec==2023.10.0 +nvidia-cusparse-cu11==11.7.4.91 +zipp==3.12.0 +mypy-extensions==1.0.0 +datasets==3.0.1 +joblib==1.3.2 +hjson==3.1.0 +traitlets==5.7.1 +stack-data==0.6.0 +transformers==4.45.1 +sympy==1.11.1 +Pygments==2.15.0 +docker-pycreds==0.4.0 +dill==0.3.8 +wheel==0.44.0 +prompt-toolkit==3.0.30 +parso==0.8.3 +ipykernel==6.23.1 +pyarrow==17.0.0 +certifi==2023.11.17 +nvidia-cufft-cu11==10.9.0.58 +six==1.16.0 +pydantic==2.9.2 +click==8.1.7 +nest-asyncio==1.5.6 +gmpy2==2.1.0 +matplotlib==3.8.2 +scipy==1.11.4 +typing_extensions==4.12.2 +statsmodels==0.14.0 +huggingface-hub==0.25.0 +frozenlist==1.4.1 +gpustat==1.1.1 +nvidia-nvtx-cu11==11.7.91 +safetensors==0.4.5 +stanza==1.9.2 +decorator==5.1.1 +seaborn==0.13.0 +sentencepiece==0.2.0 +PyYAML==6.0.1 +black==24.8.0 +protobuf==4.25.1 +pickleshare==0.7.5 +peft==0.13.0 +triton==2.0.0 +nvidia-cuda-runtime-cu11==11.7.99 +Jinja2==3.1.2 +nvidia-cusolver-cu11==11.4.0.1 +executing==1.2.0 +jupyter_client==8.1.0 +pluggy==1.3.0 +cmake==3.30.3 +pytz==2023.3.post1 +aiohappyeyeballs==2.4.2 +kiwisolver==1.4.5 +py-cpuinfo==9.0.0 +Pillow==10.1.0 +ptyprocess==0.7.0 +importlib_resources==6.4.5 +GitPython==3.1.43 +importlib-metadata==6.0.0 +iniconfig==2.0.0 +scikit-learn==1.3.2 +exceptiongroup==1.1.0 +networkx==2.8.6 +accelerate==1.0.0 +nltk==3.8.1 +shutilwhich==1.1.0 +fonttools==4.45.1 +future==0.18.3 +aiohttp==3.10.6 +wcwidth==0.2.5 +idna==3.6 +filelock==3.12.2 +pathspec==0.12.1 +jupyter_core==5.1.0 +lit==18.1.8 +nvidia-curand-cu11==10.2.10.91 +nvidia-cublas-cu11==11.10.3.66 +nvidia-ml-py==12.560.30 +msgpack==1.1.0 +python-dateutil==2.8.2 +blessed==1.20.0 +packaging==23.0 +gitdb==4.0.11 +yarl==1.13.0 +emoji==2.8.0 +tzdata==2023.3 +cycler==0.12.1 +tornado==6.2 +backcall==0.2.0 +plotnine==0.12.4 +ninja==1.11.1.1 +latex==0.7.0 +wandb==0.18.5 +setproctitle==1.3.3 +threadpoolctl==3.2.0 +requests==2.32.3 +pyparsing==3.1.1 +smmap==5.0.1 +pyzmq==23.0.0 +async-timeout==4.0.3 +annotated-types==0.7.0 +matplotlib-inline==0.1.6 +latexcodec==1.0.0 +ipython==8.0.0 +patsy==0.5.3 +contourpy==1.2.0 +multidict==6.1.0 +mizani==0.9.3 +urllib3==2.1.0 +tokenizers==0.20.0 +MarkupSafe==2.1.2 +pip==24.2 +pexpect==4.8.0 +tqdm==4.66.5 +jedi==0.18.2 +pydantic_core==2.23.4 +tempdir==0.7.1 +mpmath==1.2.1 +setuptools==72.1.0 +pytest==7.4.3 +pure-eval==0.2.2 +psutil==5.9.1 +comm==0.1.2 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +regex==2023.10.3 +platformdirs==2.5.2 +asttokens==2.2.1 +torch==2.0.0 +nvidia-nccl-cu11==2.14.3 +xxhash==3.5.0 diff --git a/wandb/run-20241101_093116-a95c3tmd/files/wandb-metadata.json b/wandb/run-20241101_093116-a95c3tmd/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..d654296ef37ef5d90d5a31def3a43cbff686f15b --- /dev/null +++ b/wandb/run-20241101_093116-a95c3tmd/files/wandb-metadata.json @@ -0,0 +1,97 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-11-01T13:31:16.545998Z", + "args": [ + "--perturbation", + "reverse_control", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "7", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", + "codePath": "train/train_deep_wandb.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_deep_wandb.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA RTX A6000", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888559353856", + "used": "1754716262400" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + } + ], + "cudaVersion": "11.8" +} \ No newline at end of file diff --git a/wandb/run-20241101_093116-a95c3tmd/logs/debug-internal.log b/wandb/run-20241101_093116-a95c3tmd/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..0f1c9135300c1d4ae12d540d0e821c14f320c298 --- /dev/null +++ b/wandb/run-20241101_093116-a95c3tmd/logs/debug-internal.log @@ -0,0 +1,8 @@ +{"time":"2024-11-01T09:31:16.5475748-04:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-11-01T09:31:16.54758968-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_093116-a95c3tmd/logs/debug-core.log"} +{"time":"2024-11-01T09:31:16.65343917-04:00","level":"INFO","msg":"created new stream","id":"a95c3tmd"} +{"time":"2024-11-01T09:31:16.65346742-04:00","level":"INFO","msg":"stream: started","id":"a95c3tmd"} +{"time":"2024-11-01T09:31:16.65353825-04:00","level":"INFO","msg":"sender: started","stream_id":"a95c3tmd"} +{"time":"2024-11-01T09:31:16.65349743-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"a95c3tmd"}} +{"time":"2024-11-01T09:31:16.6535028-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"a95c3tmd"}} +{"time":"2024-11-01T09:31:16.864150193-04:00","level":"INFO","msg":"Starting system monitor"} diff --git a/wandb/run-20241101_093116-a95c3tmd/logs/debug.log b/wandb/run-20241101_093116-a95c3tmd/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f86b5a9d168b8687eb73d241b2ce881ea8a38cce --- /dev/null +++ b/wandb/run-20241101_093116-a95c3tmd/logs/debug.log @@ -0,0 +1,26 @@ +2024-11-01 09:31:16,543 INFO MainThread:781947 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-11-01 09:31:16,543 INFO MainThread:781947 [wandb_setup.py:_flush():79] Configure stats pid to 781947 +2024-11-01 09:31:16,543 INFO MainThread:781947 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-11-01 09:31:16,543 INFO MainThread:781947 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-11-01 09:31:16,543 INFO MainThread:781947 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-01 09:31:16,543 INFO MainThread:781947 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-01 09:31:16,543 INFO MainThread:781947 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'} +2024-11-01 09:31:16,543 INFO MainThread:781947 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-01 09:31:16,543 INFO MainThread:781947 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_093116-a95c3tmd/logs/debug.log +2024-11-01 09:31:16,543 INFO MainThread:781947 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_093116-a95c3tmd/logs/debug-internal.log +2024-11-01 09:31:16,543 INFO MainThread:781947 [wandb_init.py:init():621] calling init triggers +2024-11-01 09:31:16,544 INFO MainThread:781947 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-11-01 09:31:16,544 INFO MainThread:781947 [wandb_init.py:init():671] starting backend +2024-11-01 09:31:16,544 INFO MainThread:781947 [wandb_init.py:init():675] sending inform_init request +2024-11-01 09:31:16,545 INFO MainThread:781947 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-01 09:31:16,545 INFO MainThread:781947 [wandb_init.py:init():688] backend started and connected +2024-11-01 09:31:16,548 INFO MainThread:781947 [wandb_init.py:init():783] updated telemetry +2024-11-01 09:31:16,575 INFO MainThread:781947 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-11-01 09:31:16,861 INFO MainThread:781947 [wandb_init.py:init():867] starting run threads in backend +2024-11-01 09:31:16,955 INFO MainThread:781947 [wandb_run.py:_console_start():2463] atexit reg +2024-11-01 09:31:16,955 INFO MainThread:781947 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-11-01 09:31:16,955 INFO MainThread:781947 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-11-01 09:31:16,955 INFO MainThread:781947 [wandb_run.py:_redirect():2401] Redirects installed. +2024-11-01 09:31:16,956 INFO MainThread:781947 [wandb_init.py:init():911] run started, returning control to user process +2024-11-01 09:31:16,957 INFO MainThread:781947 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_control', 'train_set': '10M', 'batch_size': 3, 'epoch': 7, 'seed': 0, 'lr': 5e-06} diff --git a/wandb/run-20241101_093116-a95c3tmd/run-a95c3tmd.wandb b/wandb/run-20241101_093116-a95c3tmd/run-a95c3tmd.wandb new file mode 100644 index 0000000000000000000000000000000000000000..eeb3ee545db7725771cc86aa4165d2b53d2cf301 Binary files /dev/null and b/wandb/run-20241101_093116-a95c3tmd/run-a95c3tmd.wandb differ diff --git a/wandb/run-20241101_201927-k1t1l8u4/files/output.log b/wandb/run-20241101_201927-k1t1l8u4/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..6302e1b298ca7176d2c07ff8db73c077397d0e90 --- /dev/null +++ b/wandb/run-20241101_201927-k1t1l8u4/files/output.log @@ -0,0 +1,13 @@ +Loading checkpoint shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:05<00:00, 2.56s/it] +tokenized_valid: Dataset({ + features: ['input_ids', 'attention_mask'], + num_rows: 600 +}) +/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +[2024-11-01 20:19:34,950] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-11-01 20:19:46,428] [INFO] [comm.py:652:init_distributed] cdb=None +Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination +Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root... +Loading extension module cpu_adam... +Time to load cpu_adam op: 5.344139575958252 seconds diff --git a/wandb/run-20241101_201927-k1t1l8u4/files/requirements.txt b/wandb/run-20241101_201927-k1t1l8u4/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..95a931302e269cc9e4fa5b719b6511f176ee2416 --- /dev/null +++ b/wandb/run-20241101_201927-k1t1l8u4/files/requirements.txt @@ -0,0 +1,147 @@ +funcsigs==1.0.2 +sentry-sdk==2.17.0 +multiprocess==0.70.16 +numpy==1.26.2 +pluralizer==1.2.0 +debugpy==1.6.7 +nvidia-cudnn-cu11==8.5.0.96 +deepspeed==0.15.2 +data==0.4 +pandas==2.1.3 +tomli==2.0.1 +charset-normalizer==3.3.2 +attrs==24.2.0 +aiosignal==1.3.1 +fsspec==2023.10.0 +nvidia-cusparse-cu11==11.7.4.91 +zipp==3.12.0 +mypy-extensions==1.0.0 +datasets==3.0.1 +joblib==1.3.2 +hjson==3.1.0 +traitlets==5.7.1 +stack-data==0.6.0 +transformers==4.45.1 +sympy==1.11.1 +Pygments==2.15.0 +docker-pycreds==0.4.0 +dill==0.3.8 +wheel==0.44.0 +prompt-toolkit==3.0.30 +parso==0.8.3 +ipykernel==6.23.1 +pyarrow==17.0.0 +certifi==2023.11.17 +nvidia-cufft-cu11==10.9.0.58 +six==1.16.0 +pydantic==2.9.2 +click==8.1.7 +nest-asyncio==1.5.6 +gmpy2==2.1.0 +matplotlib==3.8.2 +scipy==1.11.4 +typing_extensions==4.12.2 +statsmodels==0.14.0 +huggingface-hub==0.25.0 +frozenlist==1.4.1 +gpustat==1.1.1 +nvidia-nvtx-cu11==11.7.91 +safetensors==0.4.5 +stanza==1.9.2 +decorator==5.1.1 +seaborn==0.13.0 +sentencepiece==0.2.0 +PyYAML==6.0.1 +black==24.8.0 +protobuf==4.25.1 +pickleshare==0.7.5 +peft==0.13.0 +triton==2.0.0 +nvidia-cuda-runtime-cu11==11.7.99 +Jinja2==3.1.2 +nvidia-cusolver-cu11==11.4.0.1 +executing==1.2.0 +jupyter_client==8.1.0 +pluggy==1.3.0 +cmake==3.30.3 +pytz==2023.3.post1 +aiohappyeyeballs==2.4.2 +kiwisolver==1.4.5 +py-cpuinfo==9.0.0 +Pillow==10.1.0 +ptyprocess==0.7.0 +importlib_resources==6.4.5 +GitPython==3.1.43 +importlib-metadata==6.0.0 +iniconfig==2.0.0 +scikit-learn==1.3.2 +exceptiongroup==1.1.0 +networkx==2.8.6 +accelerate==1.0.0 +nltk==3.8.1 +shutilwhich==1.1.0 +fonttools==4.45.1 +future==0.18.3 +aiohttp==3.10.6 +wcwidth==0.2.5 +idna==3.6 +filelock==3.12.2 +pathspec==0.12.1 +jupyter_core==5.1.0 +lit==18.1.8 +nvidia-curand-cu11==10.2.10.91 +nvidia-cublas-cu11==11.10.3.66 +nvidia-ml-py==12.560.30 +msgpack==1.1.0 +python-dateutil==2.8.2 +blessed==1.20.0 +packaging==23.0 +gitdb==4.0.11 +yarl==1.13.0 +emoji==2.8.0 +tzdata==2023.3 +cycler==0.12.1 +tornado==6.2 +backcall==0.2.0 +plotnine==0.12.4 +ninja==1.11.1.1 +latex==0.7.0 +wandb==0.18.5 +setproctitle==1.3.3 +threadpoolctl==3.2.0 +requests==2.32.3 +pyparsing==3.1.1 +smmap==5.0.1 +pyzmq==23.0.0 +async-timeout==4.0.3 +annotated-types==0.7.0 +matplotlib-inline==0.1.6 +latexcodec==1.0.0 +ipython==8.0.0 +patsy==0.5.3 +contourpy==1.2.0 +multidict==6.1.0 +mizani==0.9.3 +urllib3==2.1.0 +tokenizers==0.20.0 +MarkupSafe==2.1.2 +pip==24.2 +pexpect==4.8.0 +tqdm==4.66.5 +jedi==0.18.2 +pydantic_core==2.23.4 +tempdir==0.7.1 +mpmath==1.2.1 +setuptools==72.1.0 +pytest==7.4.3 +pure-eval==0.2.2 +psutil==5.9.1 +comm==0.1.2 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +regex==2023.10.3 +platformdirs==2.5.2 +asttokens==2.2.1 +torch==2.0.0 +nvidia-nccl-cu11==2.14.3 +xxhash==3.5.0 diff --git a/wandb/run-20241101_201927-k1t1l8u4/files/wandb-metadata.json b/wandb/run-20241101_201927-k1t1l8u4/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..b286a924316634554443dd3c439bb85a1d82734f --- /dev/null +++ b/wandb/run-20241101_201927-k1t1l8u4/files/wandb-metadata.json @@ -0,0 +1,97 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-11-02T00:19:27.293042Z", + "args": [ + "--perturbation", + "shuffle_nondeterministic", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "3", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", + "codePath": "train/train_deep_wandb.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_deep_wandb.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA RTX A6000", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888559353856", + "used": "1754803675136" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + } + ], + "cudaVersion": "11.8" +} \ No newline at end of file diff --git a/wandb/run-20241101_201927-k1t1l8u4/logs/debug-internal.log b/wandb/run-20241101_201927-k1t1l8u4/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..445cafefee2225f456cceb242701e81577ebde58 --- /dev/null +++ b/wandb/run-20241101_201927-k1t1l8u4/logs/debug-internal.log @@ -0,0 +1,8 @@ +{"time":"2024-11-01T20:19:27.294939863-04:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-11-01T20:19:27.294953393-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_201927-k1t1l8u4/logs/debug-core.log"} +{"time":"2024-11-01T20:19:27.401466899-04:00","level":"INFO","msg":"created new stream","id":"k1t1l8u4"} +{"time":"2024-11-01T20:19:27.40151827-04:00","level":"INFO","msg":"stream: started","id":"k1t1l8u4"} +{"time":"2024-11-01T20:19:27.40160308-04:00","level":"INFO","msg":"sender: started","stream_id":"k1t1l8u4"} +{"time":"2024-11-01T20:19:27.40156729-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"k1t1l8u4"}} +{"time":"2024-11-01T20:19:27.40158983-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"k1t1l8u4"}} +{"time":"2024-11-01T20:19:27.616032711-04:00","level":"INFO","msg":"Starting system monitor"} diff --git a/wandb/run-20241101_201927-k1t1l8u4/logs/debug.log b/wandb/run-20241101_201927-k1t1l8u4/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e7c82cde2a362aa9b19fdfc09eb75a6d9c0e636e --- /dev/null +++ b/wandb/run-20241101_201927-k1t1l8u4/logs/debug.log @@ -0,0 +1,26 @@ +2024-11-01 20:19:27,291 INFO MainThread:878458 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-11-01 20:19:27,291 INFO MainThread:878458 [wandb_setup.py:_flush():79] Configure stats pid to 878458 +2024-11-01 20:19:27,291 INFO MainThread:878458 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-11-01 20:19:27,291 INFO MainThread:878458 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-11-01 20:19:27,291 INFO MainThread:878458 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-01 20:19:27,291 INFO MainThread:878458 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-01 20:19:27,291 INFO MainThread:878458 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'} +2024-11-01 20:19:27,291 INFO MainThread:878458 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-01 20:19:27,291 INFO MainThread:878458 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_201927-k1t1l8u4/logs/debug.log +2024-11-01 20:19:27,291 INFO MainThread:878458 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_201927-k1t1l8u4/logs/debug-internal.log +2024-11-01 20:19:27,291 INFO MainThread:878458 [wandb_init.py:init():621] calling init triggers +2024-11-01 20:19:27,291 INFO MainThread:878458 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-11-01 20:19:27,291 INFO MainThread:878458 [wandb_init.py:init():671] starting backend +2024-11-01 20:19:27,291 INFO MainThread:878458 [wandb_init.py:init():675] sending inform_init request +2024-11-01 20:19:27,292 INFO MainThread:878458 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-01 20:19:27,292 INFO MainThread:878458 [wandb_init.py:init():688] backend started and connected +2024-11-01 20:19:27,296 INFO MainThread:878458 [wandb_init.py:init():783] updated telemetry +2024-11-01 20:19:27,328 INFO MainThread:878458 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-11-01 20:19:27,613 INFO MainThread:878458 [wandb_init.py:init():867] starting run threads in backend +2024-11-01 20:19:27,719 INFO MainThread:878458 [wandb_run.py:_console_start():2463] atexit reg +2024-11-01 20:19:27,719 INFO MainThread:878458 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-11-01 20:19:27,719 INFO MainThread:878458 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-11-01 20:19:27,719 INFO MainThread:878458 [wandb_run.py:_redirect():2401] Redirects installed. +2024-11-01 20:19:27,721 INFO MainThread:878458 [wandb_init.py:init():911] run started, returning control to user process +2024-11-01 20:19:27,721 INFO MainThread:878458 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_nondeterministic', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0, 'lr': 5e-06} diff --git a/wandb/run-20241105_163029-2rkpz70q/files/config.yaml b/wandb/run-20241105_163029-2rkpz70q/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3771760e607bdb2dcb978de96a047ebe2475f806 --- /dev/null +++ b/wandb/run-20241105_163029-2rkpz70q/files/config.yaml @@ -0,0 +1,49 @@ +_wandb: + value: + cli_version: 0.18.5 + m: [] + python_version: 3.9.19 + t: + "1": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "2": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "3": + - 13 + - 23 + - 55 + "4": 3.9.19 + "5": 0.18.5 + "6": 4.45.1 + "8": + - 5 + "12": 0.18.5 + "13": linux-x86_64 +batch_size: + value: 3 +epoch: + value: 3 +lr: + value: 5e-06 +perturbation: + value: shuffle_deterministic57 +seed: + value: 0 +train_set: + value: 10M diff --git a/wandb/run-20241105_163029-2rkpz70q/files/output.log b/wandb/run-20241105_163029-2rkpz70q/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..9cf9af0d36a79956dc63d73c5ba1f4f946b905f4 --- /dev/null +++ b/wandb/run-20241105_163029-2rkpz70q/files/output.log @@ -0,0 +1,64 @@ +config.json: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 844/844 [00:00<00:00, 289kB/s] +model.safetensors.index.json: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████| 20.9k/20.9k [00:00<00:00, 25.5MB/s] +Downloading shards: 0%| | 0/2 [01:34 + model = AutoModelForCausalLM.from_pretrained(model_name, + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/models/auto/auto_factory.py", line 564, in from_pretrained + return model_class.from_pretrained( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/modeling_utils.py", line 3769, in from_pretrained + resolved_archive_file, sharded_metadata = get_checkpoint_shard_files( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/utils/hub.py", line 1098, in get_checkpoint_shard_files + cached_filename = cached_file( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/utils/hub.py", line 403, in cached_file + resolved_file = hf_hub_download( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/utils/_deprecation.py", line 101, in inner_f + return f(*args, **kwargs) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn + return fn(*args, **kwargs) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1232, in hf_hub_download + return _hf_hub_download_to_cache_dir( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1381, in _hf_hub_download_to_cache_dir + _download_to_tmp_and_move( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1915, in _download_to_tmp_and_move + http_get( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 558, in http_get + return http_get( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/tqdm/std.py", line 1140, in __exit__ + self.close() + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/tqdm/std.py", line 1302, in close + self.display(pos=0) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/tqdm/std.py", line 1495, in display + self.sp(self.__str__() if msg is None else msg) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/tqdm/std.py", line 1151, in __str__ + return self.format_meter(**self.format_dict) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/tqdm/std.py", line 576, in format_meter + eta_dt = (datetime.now() + timedelta(seconds=remaining) +KeyboardInterrupt diff --git a/wandb/run-20241105_163029-2rkpz70q/files/requirements.txt b/wandb/run-20241105_163029-2rkpz70q/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..95a931302e269cc9e4fa5b719b6511f176ee2416 --- /dev/null +++ b/wandb/run-20241105_163029-2rkpz70q/files/requirements.txt @@ -0,0 +1,147 @@ +funcsigs==1.0.2 +sentry-sdk==2.17.0 +multiprocess==0.70.16 +numpy==1.26.2 +pluralizer==1.2.0 +debugpy==1.6.7 +nvidia-cudnn-cu11==8.5.0.96 +deepspeed==0.15.2 +data==0.4 +pandas==2.1.3 +tomli==2.0.1 +charset-normalizer==3.3.2 +attrs==24.2.0 +aiosignal==1.3.1 +fsspec==2023.10.0 +nvidia-cusparse-cu11==11.7.4.91 +zipp==3.12.0 +mypy-extensions==1.0.0 +datasets==3.0.1 +joblib==1.3.2 +hjson==3.1.0 +traitlets==5.7.1 +stack-data==0.6.0 +transformers==4.45.1 +sympy==1.11.1 +Pygments==2.15.0 +docker-pycreds==0.4.0 +dill==0.3.8 +wheel==0.44.0 +prompt-toolkit==3.0.30 +parso==0.8.3 +ipykernel==6.23.1 +pyarrow==17.0.0 +certifi==2023.11.17 +nvidia-cufft-cu11==10.9.0.58 +six==1.16.0 +pydantic==2.9.2 +click==8.1.7 +nest-asyncio==1.5.6 +gmpy2==2.1.0 +matplotlib==3.8.2 +scipy==1.11.4 +typing_extensions==4.12.2 +statsmodels==0.14.0 +huggingface-hub==0.25.0 +frozenlist==1.4.1 +gpustat==1.1.1 +nvidia-nvtx-cu11==11.7.91 +safetensors==0.4.5 +stanza==1.9.2 +decorator==5.1.1 +seaborn==0.13.0 +sentencepiece==0.2.0 +PyYAML==6.0.1 +black==24.8.0 +protobuf==4.25.1 +pickleshare==0.7.5 +peft==0.13.0 +triton==2.0.0 +nvidia-cuda-runtime-cu11==11.7.99 +Jinja2==3.1.2 +nvidia-cusolver-cu11==11.4.0.1 +executing==1.2.0 +jupyter_client==8.1.0 +pluggy==1.3.0 +cmake==3.30.3 +pytz==2023.3.post1 +aiohappyeyeballs==2.4.2 +kiwisolver==1.4.5 +py-cpuinfo==9.0.0 +Pillow==10.1.0 +ptyprocess==0.7.0 +importlib_resources==6.4.5 +GitPython==3.1.43 +importlib-metadata==6.0.0 +iniconfig==2.0.0 +scikit-learn==1.3.2 +exceptiongroup==1.1.0 +networkx==2.8.6 +accelerate==1.0.0 +nltk==3.8.1 +shutilwhich==1.1.0 +fonttools==4.45.1 +future==0.18.3 +aiohttp==3.10.6 +wcwidth==0.2.5 +idna==3.6 +filelock==3.12.2 +pathspec==0.12.1 +jupyter_core==5.1.0 +lit==18.1.8 +nvidia-curand-cu11==10.2.10.91 +nvidia-cublas-cu11==11.10.3.66 +nvidia-ml-py==12.560.30 +msgpack==1.1.0 +python-dateutil==2.8.2 +blessed==1.20.0 +packaging==23.0 +gitdb==4.0.11 +yarl==1.13.0 +emoji==2.8.0 +tzdata==2023.3 +cycler==0.12.1 +tornado==6.2 +backcall==0.2.0 +plotnine==0.12.4 +ninja==1.11.1.1 +latex==0.7.0 +wandb==0.18.5 +setproctitle==1.3.3 +threadpoolctl==3.2.0 +requests==2.32.3 +pyparsing==3.1.1 +smmap==5.0.1 +pyzmq==23.0.0 +async-timeout==4.0.3 +annotated-types==0.7.0 +matplotlib-inline==0.1.6 +latexcodec==1.0.0 +ipython==8.0.0 +patsy==0.5.3 +contourpy==1.2.0 +multidict==6.1.0 +mizani==0.9.3 +urllib3==2.1.0 +tokenizers==0.20.0 +MarkupSafe==2.1.2 +pip==24.2 +pexpect==4.8.0 +tqdm==4.66.5 +jedi==0.18.2 +pydantic_core==2.23.4 +tempdir==0.7.1 +mpmath==1.2.1 +setuptools==72.1.0 +pytest==7.4.3 +pure-eval==0.2.2 +psutil==5.9.1 +comm==0.1.2 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +regex==2023.10.3 +platformdirs==2.5.2 +asttokens==2.2.1 +torch==2.0.0 +nvidia-nccl-cu11==2.14.3 +xxhash==3.5.0 diff --git a/wandb/run-20241105_163029-2rkpz70q/files/wandb-metadata.json b/wandb/run-20241105_163029-2rkpz70q/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..84780a4ba3e55c7b8387aafceea69b7964772e56 --- /dev/null +++ b/wandb/run-20241105_163029-2rkpz70q/files/wandb-metadata.json @@ -0,0 +1,97 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-11-05T21:30:29.293234Z", + "args": [ + "--perturbation", + "shuffle_deterministic57", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "3", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", + "codePath": "train/train_deep_wandb.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_deep_wandb.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA RTX A6000", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888559353856", + "used": "1785984946176" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + } + ], + "cudaVersion": "11.8" +} \ No newline at end of file diff --git a/wandb/run-20241105_163029-2rkpz70q/files/wandb-summary.json b/wandb/run-20241105_163029-2rkpz70q/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..04c3323e52593a41b78537061e13cb2123c27d21 --- /dev/null +++ b/wandb/run-20241105_163029-2rkpz70q/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":95}} \ No newline at end of file diff --git a/wandb/run-20241105_163029-2rkpz70q/logs/debug-internal.log b/wandb/run-20241105_163029-2rkpz70q/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..638458b245d7624b61011ae2aade1cd9b75dd131 --- /dev/null +++ b/wandb/run-20241105_163029-2rkpz70q/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2024-11-05T16:30:29.296088295-05:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-11-05T16:30:29.296109055-05:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_163029-2rkpz70q/logs/debug-core.log"} +{"time":"2024-11-05T16:30:29.403034023-05:00","level":"INFO","msg":"created new stream","id":"2rkpz70q"} +{"time":"2024-11-05T16:30:29.403087833-05:00","level":"INFO","msg":"stream: started","id":"2rkpz70q"} +{"time":"2024-11-05T16:30:29.403122053-05:00","level":"INFO","msg":"sender: started","stream_id":"2rkpz70q"} +{"time":"2024-11-05T16:30:29.403116283-05:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"2rkpz70q"}} +{"time":"2024-11-05T16:30:29.403141933-05:00","level":"INFO","msg":"handler: started","stream_id":{"value":"2rkpz70q"}} +{"time":"2024-11-05T16:30:29.655903657-05:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-11-05T16:32:04.673881802-05:00","level":"INFO","msg":"stream: closing","id":"2rkpz70q"} +{"time":"2024-11-05T16:32:04.673919782-05:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2024-11-05T16:32:04.674440985-05:00","level":"INFO","msg":"Stopped system monitor"} diff --git a/wandb/run-20241105_163029-2rkpz70q/logs/debug.log b/wandb/run-20241105_163029-2rkpz70q/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..a0200e44cb6359d41421f7d1f91a3aafa2e9c91a --- /dev/null +++ b/wandb/run-20241105_163029-2rkpz70q/logs/debug.log @@ -0,0 +1,27 @@ +2024-11-05 16:30:29,290 INFO MainThread:1779704 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-11-05 16:30:29,290 INFO MainThread:1779704 [wandb_setup.py:_flush():79] Configure stats pid to 1779704 +2024-11-05 16:30:29,290 INFO MainThread:1779704 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-11-05 16:30:29,290 INFO MainThread:1779704 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-11-05 16:30:29,290 INFO MainThread:1779704 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-05 16:30:29,290 INFO MainThread:1779704 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-05 16:30:29,290 INFO MainThread:1779704 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'} +2024-11-05 16:30:29,290 INFO MainThread:1779704 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-05 16:30:29,290 INFO MainThread:1779704 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_163029-2rkpz70q/logs/debug.log +2024-11-05 16:30:29,290 INFO MainThread:1779704 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_163029-2rkpz70q/logs/debug-internal.log +2024-11-05 16:30:29,290 INFO MainThread:1779704 [wandb_init.py:init():621] calling init triggers +2024-11-05 16:30:29,290 INFO MainThread:1779704 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-11-05 16:30:29,290 INFO MainThread:1779704 [wandb_init.py:init():671] starting backend +2024-11-05 16:30:29,290 INFO MainThread:1779704 [wandb_init.py:init():675] sending inform_init request +2024-11-05 16:30:29,292 INFO MainThread:1779704 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-05 16:30:29,292 INFO MainThread:1779704 [wandb_init.py:init():688] backend started and connected +2024-11-05 16:30:29,297 INFO MainThread:1779704 [wandb_init.py:init():783] updated telemetry +2024-11-05 16:30:29,329 INFO MainThread:1779704 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-11-05 16:30:29,653 INFO MainThread:1779704 [wandb_init.py:init():867] starting run threads in backend +2024-11-05 16:30:29,738 INFO MainThread:1779704 [wandb_run.py:_console_start():2463] atexit reg +2024-11-05 16:30:29,738 INFO MainThread:1779704 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-11-05 16:30:29,739 INFO MainThread:1779704 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-11-05 16:30:29,739 INFO MainThread:1779704 [wandb_run.py:_redirect():2401] Redirects installed. +2024-11-05 16:30:29,740 INFO MainThread:1779704 [wandb_init.py:init():911] run started, returning control to user process +2024-11-05 16:30:29,740 INFO MainThread:1779704 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_deterministic57', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0, 'lr': 5e-06} +2024-11-05 16:32:04,674 WARNING MsgRouterThr:1779704 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20241105_163029-ccve22u0/files/config.yaml b/wandb/run-20241105_163029-ccve22u0/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3771760e607bdb2dcb978de96a047ebe2475f806 --- /dev/null +++ b/wandb/run-20241105_163029-ccve22u0/files/config.yaml @@ -0,0 +1,49 @@ +_wandb: + value: + cli_version: 0.18.5 + m: [] + python_version: 3.9.19 + t: + "1": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "2": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "3": + - 13 + - 23 + - 55 + "4": 3.9.19 + "5": 0.18.5 + "6": 4.45.1 + "8": + - 5 + "12": 0.18.5 + "13": linux-x86_64 +batch_size: + value: 3 +epoch: + value: 3 +lr: + value: 5e-06 +perturbation: + value: shuffle_deterministic57 +seed: + value: 0 +train_set: + value: 10M diff --git a/wandb/run-20241105_163029-ccve22u0/files/output.log b/wandb/run-20241105_163029-ccve22u0/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..02622994229eeebbb6e354bc6cf8155dd9b2d08f --- /dev/null +++ b/wandb/run-20241105_163029-ccve22u0/files/output.log @@ -0,0 +1,37 @@ +Downloading shards: 0%| | 0/2 [00:00 +Traceback (most recent call last): + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/tqdm/std.py", line 1196, in __iter__ + self.close() + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/tqdm/std.py", line 1302, in close + self.display(pos=0) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/tqdm/std.py", line 1495, in display + self.sp(self.__str__() if msg is None else msg) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/tqdm/std.py", line 459, in print_status + fp_write('\r' + s + (' ' * max(last_len[0] - len_s, 0))) +KeyboardInterrupt: +Traceback (most recent call last): + File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 174, in + model = AutoModelForCausalLM.from_pretrained(model_name, + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/models/auto/auto_factory.py", line 564, in from_pretrained + return model_class.from_pretrained( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/modeling_utils.py", line 3769, in from_pretrained + resolved_archive_file, sharded_metadata = get_checkpoint_shard_files( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/utils/hub.py", line 1098, in get_checkpoint_shard_files + cached_filename = cached_file( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/utils/hub.py", line 403, in cached_file + resolved_file = hf_hub_download( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/utils/_deprecation.py", line 101, in inner_f + return f(*args, **kwargs) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn + return fn(*args, **kwargs) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1232, in hf_hub_download + return _hf_hub_download_to_cache_dir( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1380, in _hf_hub_download_to_cache_dir + with WeakFileLock(lock_path): + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/contextlib.py", line 119, in __enter__ + return next(self.gen) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/utils/_fixes.py", line 98, in WeakFileLock + lock.acquire() + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/filelock/_api.py", line 225, in acquire + time.sleep(poll_interval) +KeyboardInterrupt diff --git a/wandb/run-20241105_163029-ccve22u0/files/requirements.txt b/wandb/run-20241105_163029-ccve22u0/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..95a931302e269cc9e4fa5b719b6511f176ee2416 --- /dev/null +++ b/wandb/run-20241105_163029-ccve22u0/files/requirements.txt @@ -0,0 +1,147 @@ +funcsigs==1.0.2 +sentry-sdk==2.17.0 +multiprocess==0.70.16 +numpy==1.26.2 +pluralizer==1.2.0 +debugpy==1.6.7 +nvidia-cudnn-cu11==8.5.0.96 +deepspeed==0.15.2 +data==0.4 +pandas==2.1.3 +tomli==2.0.1 +charset-normalizer==3.3.2 +attrs==24.2.0 +aiosignal==1.3.1 +fsspec==2023.10.0 +nvidia-cusparse-cu11==11.7.4.91 +zipp==3.12.0 +mypy-extensions==1.0.0 +datasets==3.0.1 +joblib==1.3.2 +hjson==3.1.0 +traitlets==5.7.1 +stack-data==0.6.0 +transformers==4.45.1 +sympy==1.11.1 +Pygments==2.15.0 +docker-pycreds==0.4.0 +dill==0.3.8 +wheel==0.44.0 +prompt-toolkit==3.0.30 +parso==0.8.3 +ipykernel==6.23.1 +pyarrow==17.0.0 +certifi==2023.11.17 +nvidia-cufft-cu11==10.9.0.58 +six==1.16.0 +pydantic==2.9.2 +click==8.1.7 +nest-asyncio==1.5.6 +gmpy2==2.1.0 +matplotlib==3.8.2 +scipy==1.11.4 +typing_extensions==4.12.2 +statsmodels==0.14.0 +huggingface-hub==0.25.0 +frozenlist==1.4.1 +gpustat==1.1.1 +nvidia-nvtx-cu11==11.7.91 +safetensors==0.4.5 +stanza==1.9.2 +decorator==5.1.1 +seaborn==0.13.0 +sentencepiece==0.2.0 +PyYAML==6.0.1 +black==24.8.0 +protobuf==4.25.1 +pickleshare==0.7.5 +peft==0.13.0 +triton==2.0.0 +nvidia-cuda-runtime-cu11==11.7.99 +Jinja2==3.1.2 +nvidia-cusolver-cu11==11.4.0.1 +executing==1.2.0 +jupyter_client==8.1.0 +pluggy==1.3.0 +cmake==3.30.3 +pytz==2023.3.post1 +aiohappyeyeballs==2.4.2 +kiwisolver==1.4.5 +py-cpuinfo==9.0.0 +Pillow==10.1.0 +ptyprocess==0.7.0 +importlib_resources==6.4.5 +GitPython==3.1.43 +importlib-metadata==6.0.0 +iniconfig==2.0.0 +scikit-learn==1.3.2 +exceptiongroup==1.1.0 +networkx==2.8.6 +accelerate==1.0.0 +nltk==3.8.1 +shutilwhich==1.1.0 +fonttools==4.45.1 +future==0.18.3 +aiohttp==3.10.6 +wcwidth==0.2.5 +idna==3.6 +filelock==3.12.2 +pathspec==0.12.1 +jupyter_core==5.1.0 +lit==18.1.8 +nvidia-curand-cu11==10.2.10.91 +nvidia-cublas-cu11==11.10.3.66 +nvidia-ml-py==12.560.30 +msgpack==1.1.0 +python-dateutil==2.8.2 +blessed==1.20.0 +packaging==23.0 +gitdb==4.0.11 +yarl==1.13.0 +emoji==2.8.0 +tzdata==2023.3 +cycler==0.12.1 +tornado==6.2 +backcall==0.2.0 +plotnine==0.12.4 +ninja==1.11.1.1 +latex==0.7.0 +wandb==0.18.5 +setproctitle==1.3.3 +threadpoolctl==3.2.0 +requests==2.32.3 +pyparsing==3.1.1 +smmap==5.0.1 +pyzmq==23.0.0 +async-timeout==4.0.3 +annotated-types==0.7.0 +matplotlib-inline==0.1.6 +latexcodec==1.0.0 +ipython==8.0.0 +patsy==0.5.3 +contourpy==1.2.0 +multidict==6.1.0 +mizani==0.9.3 +urllib3==2.1.0 +tokenizers==0.20.0 +MarkupSafe==2.1.2 +pip==24.2 +pexpect==4.8.0 +tqdm==4.66.5 +jedi==0.18.2 +pydantic_core==2.23.4 +tempdir==0.7.1 +mpmath==1.2.1 +setuptools==72.1.0 +pytest==7.4.3 +pure-eval==0.2.2 +psutil==5.9.1 +comm==0.1.2 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +regex==2023.10.3 +platformdirs==2.5.2 +asttokens==2.2.1 +torch==2.0.0 +nvidia-nccl-cu11==2.14.3 +xxhash==3.5.0 diff --git a/wandb/run-20241105_163029-ccve22u0/files/wandb-metadata.json b/wandb/run-20241105_163029-ccve22u0/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..b0d0442d7c4900b1b655f67a1b27c7cbdb420165 --- /dev/null +++ b/wandb/run-20241105_163029-ccve22u0/files/wandb-metadata.json @@ -0,0 +1,97 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-11-05T21:30:29.509669Z", + "args": [ + "--perturbation", + "shuffle_deterministic57", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "3", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", + "codePath": "train/train_deep_wandb.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_deep_wandb.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA RTX A6000", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888559353856", + "used": "1785984946176" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + } + ], + "cudaVersion": "11.8" +} \ No newline at end of file diff --git a/wandb/run-20241105_163029-ccve22u0/files/wandb-summary.json b/wandb/run-20241105_163029-ccve22u0/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..04c3323e52593a41b78537061e13cb2123c27d21 --- /dev/null +++ b/wandb/run-20241105_163029-ccve22u0/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":95}} \ No newline at end of file diff --git a/wandb/run-20241105_163029-ccve22u0/logs/debug-internal.log b/wandb/run-20241105_163029-ccve22u0/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..09f9f4c374565ed215ec38601c11597fb572c57a --- /dev/null +++ b/wandb/run-20241105_163029-ccve22u0/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2024-11-05T16:30:29.512459431-05:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-11-05T16:30:29.512479911-05:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_163029-ccve22u0/logs/debug-core.log"} +{"time":"2024-11-05T16:30:29.62225023-05:00","level":"INFO","msg":"created new stream","id":"ccve22u0"} +{"time":"2024-11-05T16:30:29.62231132-05:00","level":"INFO","msg":"stream: started","id":"ccve22u0"} +{"time":"2024-11-05T16:30:29.62239515-05:00","level":"INFO","msg":"sender: started","stream_id":"ccve22u0"} +{"time":"2024-11-05T16:30:29.62237429-05:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"ccve22u0"}} +{"time":"2024-11-05T16:30:29.62244178-05:00","level":"INFO","msg":"handler: started","stream_id":{"value":"ccve22u0"}} +{"time":"2024-11-05T16:30:29.893368769-05:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-11-05T16:32:04.674804536-05:00","level":"INFO","msg":"stream: closing","id":"ccve22u0"} +{"time":"2024-11-05T16:32:04.674852526-05:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2024-11-05T16:32:04.676060992-05:00","level":"INFO","msg":"Stopped system monitor"} diff --git a/wandb/run-20241105_163029-ccve22u0/logs/debug.log b/wandb/run-20241105_163029-ccve22u0/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..5f4fb279610ea1e176208605a4a605d3af3bdfe6 --- /dev/null +++ b/wandb/run-20241105_163029-ccve22u0/logs/debug.log @@ -0,0 +1,27 @@ +2024-11-05 16:30:29,506 INFO MainThread:1779703 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-11-05 16:30:29,506 INFO MainThread:1779703 [wandb_setup.py:_flush():79] Configure stats pid to 1779703 +2024-11-05 16:30:29,506 INFO MainThread:1779703 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-11-05 16:30:29,506 INFO MainThread:1779703 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-11-05 16:30:29,506 INFO MainThread:1779703 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-05 16:30:29,506 INFO MainThread:1779703 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-05 16:30:29,506 INFO MainThread:1779703 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'} +2024-11-05 16:30:29,506 INFO MainThread:1779703 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-05 16:30:29,507 INFO MainThread:1779703 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_163029-ccve22u0/logs/debug.log +2024-11-05 16:30:29,507 INFO MainThread:1779703 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_163029-ccve22u0/logs/debug-internal.log +2024-11-05 16:30:29,507 INFO MainThread:1779703 [wandb_init.py:init():621] calling init triggers +2024-11-05 16:30:29,507 INFO MainThread:1779703 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-11-05 16:30:29,507 INFO MainThread:1779703 [wandb_init.py:init():671] starting backend +2024-11-05 16:30:29,507 INFO MainThread:1779703 [wandb_init.py:init():675] sending inform_init request +2024-11-05 16:30:29,509 INFO MainThread:1779703 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-05 16:30:29,509 INFO MainThread:1779703 [wandb_init.py:init():688] backend started and connected +2024-11-05 16:30:29,512 INFO MainThread:1779703 [wandb_init.py:init():783] updated telemetry +2024-11-05 16:30:29,549 INFO MainThread:1779703 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-11-05 16:30:29,888 INFO MainThread:1779703 [wandb_init.py:init():867] starting run threads in backend +2024-11-05 16:30:29,980 INFO MainThread:1779703 [wandb_run.py:_console_start():2463] atexit reg +2024-11-05 16:30:29,980 INFO MainThread:1779703 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-11-05 16:30:29,980 INFO MainThread:1779703 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-11-05 16:30:29,980 INFO MainThread:1779703 [wandb_run.py:_redirect():2401] Redirects installed. +2024-11-05 16:30:29,982 INFO MainThread:1779703 [wandb_init.py:init():911] run started, returning control to user process +2024-11-05 16:30:29,982 INFO MainThread:1779703 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_deterministic57', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0, 'lr': 5e-06} +2024-11-05 16:32:04,674 WARNING MsgRouterThr:1779703 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20241105_163029-ccve22u0/run-ccve22u0.wandb b/wandb/run-20241105_163029-ccve22u0/run-ccve22u0.wandb new file mode 100644 index 0000000000000000000000000000000000000000..d0301194beb501376f76030dad2919a04adcbdf3 Binary files /dev/null and b/wandb/run-20241105_163029-ccve22u0/run-ccve22u0.wandb differ diff --git a/wandb/run-20241105_163244-ax7d1x4c/files/output.log b/wandb/run-20241105_163244-ax7d1x4c/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..8a4ca116b913e849535623efcfe48fa7bd454159 --- /dev/null +++ b/wandb/run-20241105_163244-ax7d1x4c/files/output.log @@ -0,0 +1,852 @@ +model-00001-of-00002.safetensors: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████| 4.97G/4.97G [00:24<00:00, 41.5MB/s] +model-00002-of-00002.safetensors: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████| 1.46G/1.46G [00:34<00:00, 41.8MB/s] +Downloading shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:59<00:00, 29.62s/it] +Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:04<00:00, 2.14s/it] +generation_config.json: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 185/185 [00:00<00:00, 49.6kB/s] +Map: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 16425/16425 [00:45<00:00, 359.25 examples/s] +Map: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 17013/17013 [00:46<00:00, 363.96 examples/s] +tokenized_valid: Dataset({ + features: ['input_ids', 'attention_mask'], + num_rows: 1000 +}) +/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +[2024-11-05 16:35:22,821] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-11-05 16:35:30,976] [INFO] [comm.py:652:init_distributed] cdb=None +[2024-11-05 16:35:30,976] [INFO] [comm.py:683:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl +Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher. +Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination +Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root... +Emitting ninja build file /home/chunhui/.cache/torch_extensions/py39_cu117/cpu_adam/build.ninja... +Building extension module cpu_adam... +Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +Loading extension module cpu_adam... +Time to load cpu_adam op: 4.831990003585815 seconds +wandb: WARNING The `run_name` is currently set to the same value as `TrainingArguments.output_dir`. If this was not intended, please specify a different run name by setting the `TrainingArguments.run_name` parameter. + +{'loss': 3.0566, 'grad_norm': 0.0, 'learning_rate': 0.0, 'epoch': 0.0} +{'loss': 3.045, 'grad_norm': 0.0, 'learning_rate': 0.0, 'epoch': 0.0} +{'loss': 3.0231, 'grad_norm': 0.0, 'learning_rate': 0.0, 'epoch': 0.0} +{'loss': 3.0369, 'grad_norm': 0.0, 'learning_rate': 0.0, 'epoch': 0.0} +{'loss': 3.0178, 'grad_norm': 0.0, 'learning_rate': 0.0, 'epoch': 0.01} +{'loss': 3.1366, 'grad_norm': 0.0, 'learning_rate': 0.0, 'epoch': 0.01} +{'loss': 3.0555, 'grad_norm': 0.0, 'learning_rate': 0.0, 'epoch': 0.01} +{'loss': 3.094, 'grad_norm': 0.0, 'learning_rate': 0.0, 'epoch': 0.01} +{'loss': 3.0471, 'grad_norm': 0.0, 'learning_rate': 0.0, 'epoch': 0.01} +{'loss': 3.1006, 'grad_norm': 0.0, 'learning_rate': 0.0, 'epoch': 0.01} + +{'eval_loss': 3.0924079418182373, 'eval_runtime': 38.0753, 'eval_samples_per_second': 26.264, 'eval_steps_per_second': 1.103, 'epoch': 0.01} +{'loss': 3.0849, 'grad_norm': 0.0, 'learning_rate': 0.0, 'epoch': 0.01} +{'loss': 2.9997, 'grad_norm': 0.0, 'learning_rate': 0.0, 'epoch': 0.01} +{'loss': 3.0669, 'grad_norm': 0.0, 'learning_rate': 0.0, 'epoch': 0.01} +{'loss': 3.0588, 'grad_norm': 0.0, 'learning_rate': 0.0, 'epoch': 0.02} +{'loss': 3.0983, 'grad_norm': 5.574987411499023, 'learning_rate': 1.8248175182481753e-08, 'epoch': 0.02} +{'loss': 3.0508, 'grad_norm': 6.172506809234619, 'learning_rate': 3.6496350364963505e-08, 'epoch': 0.02} +{'loss': 3.0467, 'grad_norm': 6.234955787658691, 'learning_rate': 5.4744525547445264e-08, 'epoch': 0.02} +{'loss': 3.0846, 'grad_norm': 6.437706470489502, 'learning_rate': 7.299270072992701e-08, 'epoch': 0.02} +{'loss': 3.0289, 'grad_norm': 6.482471466064453, 'learning_rate': 9.124087591240876e-08, 'epoch': 0.02} +{'loss': 3.1008, 'grad_norm': 5.701765060424805, 'learning_rate': 1.0948905109489053e-07, 'epoch': 0.02} +{'eval_loss': 3.0910158157348633, 'eval_runtime': 37.1471, 'eval_samples_per_second': 26.92, 'eval_steps_per_second': 1.131, 'epoch': 0.02} +{'loss': 3.0819, 'grad_norm': 6.222895622253418, 'learning_rate': 1.2773722627737227e-07, 'epoch': 0.02} +{'loss': 3.0046, 'grad_norm': 6.059528827667236, 'learning_rate': 1.4598540145985402e-07, 'epoch': 0.02} +{'loss': 3.0459, 'grad_norm': 5.951301574707031, 'learning_rate': 1.642335766423358e-07, 'epoch': 0.03} +{'loss': 3.0823, 'grad_norm': 5.824173927307129, 'learning_rate': 1.8248175182481753e-07, 'epoch': 0.03} +{'loss': 3.0828, 'grad_norm': 5.564741134643555, 'learning_rate': 2.0072992700729928e-07, 'epoch': 0.03} +{'loss': 3.0587, 'grad_norm': 4.622601509094238, 'learning_rate': 2.1897810218978106e-07, 'epoch': 0.03} +{'loss': 3.0741, 'grad_norm': 4.757779121398926, 'learning_rate': 2.3722627737226278e-07, 'epoch': 0.03} +{'loss': 3.0245, 'grad_norm': 4.821556568145752, 'learning_rate': 2.5547445255474454e-07, 'epoch': 0.03} +{'loss': 3.0195, 'grad_norm': 4.55632209777832, 'learning_rate': 2.737226277372263e-07, 'epoch': 0.03} +{'loss': 2.9946, 'grad_norm': 4.604625701904297, 'learning_rate': 2.9197080291970804e-07, 'epoch': 0.03} +{'eval_loss': 3.0480520725250244, 'eval_runtime': 36.8258, 'eval_samples_per_second': 27.155, 'eval_steps_per_second': 1.141, 'epoch': 0.03} +{'loss': 3.0171, 'grad_norm': 4.086721897125244, 'learning_rate': 3.102189781021898e-07, 'epoch': 0.03} +{'loss': 2.9961, 'grad_norm': 4.214878559112549, 'learning_rate': 3.284671532846716e-07, 'epoch': 0.04} +{'loss': 2.9751, 'grad_norm': 4.155057907104492, 'learning_rate': 3.467153284671533e-07, 'epoch': 0.04} +{'loss': 2.9689, 'grad_norm': 3.9290964603424072, 'learning_rate': 3.6496350364963505e-07, 'epoch': 0.04} +{'loss': 2.9864, 'grad_norm': 4.033408164978027, 'learning_rate': 3.8321167883211686e-07, 'epoch': 0.04} +{'loss': 3.0125, 'grad_norm': 3.602386474609375, 'learning_rate': 4.0145985401459856e-07, 'epoch': 0.04} +{'loss': 2.9467, 'grad_norm': 3.1355831623077393, 'learning_rate': 4.197080291970803e-07, 'epoch': 0.04} +{'loss': 2.9117, 'grad_norm': 3.539684295654297, 'learning_rate': 4.379562043795621e-07, 'epoch': 0.04} +{'loss': 3.0009, 'grad_norm': 3.389564037322998, 'learning_rate': 4.562043795620438e-07, 'epoch': 0.04} +{'loss': 2.9096, 'grad_norm': 3.539494752883911, 'learning_rate': 4.7445255474452557e-07, 'epoch': 0.04} +{'eval_loss': 2.973116397857666, 'eval_runtime': 36.7932, 'eval_samples_per_second': 27.179, 'eval_steps_per_second': 1.142, 'epoch': 0.04} +{'loss': 2.9378, 'grad_norm': 3.2061192989349365, 'learning_rate': 4.927007299270073e-07, 'epoch': 0.04} +{'loss': 2.9661, 'grad_norm': 2.979443311691284, 'learning_rate': 5.109489051094891e-07, 'epoch': 0.05} +{'loss': 2.9306, 'grad_norm': 3.0862207412719727, 'learning_rate': 5.291970802919708e-07, 'epoch': 0.05} +{'loss': 2.9484, 'grad_norm': 3.0377442836761475, 'learning_rate': 5.474452554744526e-07, 'epoch': 0.05} +{'loss': 2.9258, 'grad_norm': 2.934086799621582, 'learning_rate': 5.656934306569343e-07, 'epoch': 0.05} +{'loss': 2.8798, 'grad_norm': 2.9917893409729004, 'learning_rate': 5.839416058394161e-07, 'epoch': 0.05} +{'loss': 2.8665, 'grad_norm': 2.9639346599578857, 'learning_rate': 6.021897810218979e-07, 'epoch': 0.05} +{'loss': 2.8879, 'grad_norm': 2.7003400325775146, 'learning_rate': 6.204379562043796e-07, 'epoch': 0.05} +{'loss': 2.8109, 'grad_norm': 2.8495047092437744, 'learning_rate': 6.386861313868614e-07, 'epoch': 0.05} +{'loss': 2.929, 'grad_norm': 2.1759133338928223, 'learning_rate': 6.569343065693432e-07, 'epoch': 0.05} +{'eval_loss': 2.884573221206665, 'eval_runtime': 36.9227, 'eval_samples_per_second': 27.084, 'eval_steps_per_second': 1.138, 'epoch': 0.05} +{'loss': 2.8279, 'grad_norm': 2.7880029678344727, 'learning_rate': 6.751824817518248e-07, 'epoch': 0.06} +{'loss': 2.8474, 'grad_norm': 2.3388028144836426, 'learning_rate': 6.934306569343066e-07, 'epoch': 0.06} +{'loss': 2.7775, 'grad_norm': 2.34242844581604, 'learning_rate': 7.116788321167883e-07, 'epoch': 0.06} +{'loss': 2.7831, 'grad_norm': 2.2962350845336914, 'learning_rate': 7.299270072992701e-07, 'epoch': 0.06} +{'loss': 2.7994, 'grad_norm': 2.353131055831909, 'learning_rate': 7.48175182481752e-07, 'epoch': 0.06} +{'loss': 2.8155, 'grad_norm': 2.1646981239318848, 'learning_rate': 7.664233576642337e-07, 'epoch': 0.06} +{'loss': 2.8154, 'grad_norm': 2.34967303276062, 'learning_rate': 7.846715328467154e-07, 'epoch': 0.06} +{'loss': 2.7908, 'grad_norm': 2.1275205612182617, 'learning_rate': 8.029197080291971e-07, 'epoch': 0.06} +{'loss': 2.7442, 'grad_norm': 1.89387047290802, 'learning_rate': 8.211678832116789e-07, 'epoch': 0.06} +{'loss': 2.7103, 'grad_norm': 1.8902339935302734, 'learning_rate': 8.394160583941606e-07, 'epoch': 0.07} +{'eval_loss': 2.815255641937256, 'eval_runtime': 36.8785, 'eval_samples_per_second': 27.116, 'eval_steps_per_second': 1.139, 'epoch': 0.07} +{'loss': 2.7477, 'grad_norm': 2.1961512565612793, 'learning_rate': 8.576642335766425e-07, 'epoch': 0.07} +{'loss': 2.7376, 'grad_norm': 1.8918277025222778, 'learning_rate': 8.759124087591242e-07, 'epoch': 0.07} +{'loss': 2.7539, 'grad_norm': 1.9290616512298584, 'learning_rate': 8.94160583941606e-07, 'epoch': 0.07} +{'loss': 2.7609, 'grad_norm': 1.7825456857681274, 'learning_rate': 9.124087591240876e-07, 'epoch': 0.07} +{'loss': 2.7161, 'grad_norm': 2.140242576599121, 'learning_rate': 9.306569343065694e-07, 'epoch': 0.07} +{'loss': 2.7358, 'grad_norm': 2.2971854209899902, 'learning_rate': 9.489051094890511e-07, 'epoch': 0.07} +{'loss': 2.7235, 'grad_norm': 1.9837063550949097, 'learning_rate': 9.67153284671533e-07, 'epoch': 0.07} +{'loss': 2.771, 'grad_norm': 2.1990857124328613, 'learning_rate': 9.854014598540146e-07, 'epoch': 0.07} +{'loss': 2.6975, 'grad_norm': 2.3288135528564453, 'learning_rate': 1.0036496350364965e-06, 'epoch': 0.08} +{'loss': 2.6627, 'grad_norm': 2.0509581565856934, 'learning_rate': 1.0218978102189781e-06, 'epoch': 0.08} +{'eval_loss': 2.743861675262451, 'eval_runtime': 36.9626, 'eval_samples_per_second': 27.054, 'eval_steps_per_second': 1.136, 'epoch': 0.08} +{'loss': 2.7297, 'grad_norm': 2.070568561553955, 'learning_rate': 1.04014598540146e-06, 'epoch': 0.08} +{'loss': 2.6152, 'grad_norm': 2.5551228523254395, 'learning_rate': 1.0583941605839416e-06, 'epoch': 0.08} +{'loss': 2.6144, 'grad_norm': 2.060121536254883, 'learning_rate': 1.0766423357664233e-06, 'epoch': 0.08} +{'loss': 2.6374, 'grad_norm': 2.783064365386963, 'learning_rate': 1.0948905109489052e-06, 'epoch': 0.08} +{'loss': 2.661, 'grad_norm': 2.765652894973755, 'learning_rate': 1.113138686131387e-06, 'epoch': 0.08} +{'loss': 2.666, 'grad_norm': 2.0037786960601807, 'learning_rate': 1.1313868613138687e-06, 'epoch': 0.08} +{'loss': 2.5964, 'grad_norm': 2.9089527130126953, 'learning_rate': 1.1496350364963505e-06, 'epoch': 0.08} +{'loss': 2.5754, 'grad_norm': 2.1275577545166016, 'learning_rate': 1.1678832116788322e-06, 'epoch': 0.09} +{'loss': 2.585, 'grad_norm': 2.3247408866882324, 'learning_rate': 1.1861313868613138e-06, 'epoch': 0.09} +{'loss': 2.5757, 'grad_norm': 2.5372109413146973, 'learning_rate': 1.2043795620437959e-06, 'epoch': 0.09} +{'eval_loss': 2.664235830307007, 'eval_runtime': 36.9631, 'eval_samples_per_second': 27.054, 'eval_steps_per_second': 1.136, 'epoch': 0.09} +{'loss': 2.581, 'grad_norm': 2.0754737854003906, 'learning_rate': 1.2226277372262775e-06, 'epoch': 0.09} +{'loss': 2.6194, 'grad_norm': 2.7675907611846924, 'learning_rate': 1.2408759124087592e-06, 'epoch': 0.09} +{'loss': 2.5604, 'grad_norm': 2.2297277450561523, 'learning_rate': 1.259124087591241e-06, 'epoch': 0.09} +{'loss': 2.5737, 'grad_norm': 2.3006038665771484, 'learning_rate': 1.2773722627737229e-06, 'epoch': 0.09} +{'loss': 2.5599, 'grad_norm': 2.127249002456665, 'learning_rate': 1.2956204379562045e-06, 'epoch': 0.09} +{'loss': 2.5135, 'grad_norm': 2.4705958366394043, 'learning_rate': 1.3138686131386864e-06, 'epoch': 0.09} +{'loss': 2.5267, 'grad_norm': 2.6609578132629395, 'learning_rate': 1.332116788321168e-06, 'epoch': 0.1} +{'loss': 2.4526, 'grad_norm': 2.690622091293335, 'learning_rate': 1.3503649635036497e-06, 'epoch': 0.1} +{'loss': 2.4935, 'grad_norm': 2.703291893005371, 'learning_rate': 1.3686131386861315e-06, 'epoch': 0.1} +{'loss': 2.425, 'grad_norm': 2.3661017417907715, 'learning_rate': 1.3868613138686132e-06, 'epoch': 0.1} +{'eval_loss': 2.579646110534668, 'eval_runtime': 37.0218, 'eval_samples_per_second': 27.011, 'eval_steps_per_second': 1.134, 'epoch': 0.1} +{'loss': 2.5244, 'grad_norm': 2.596808910369873, 'learning_rate': 1.405109489051095e-06, 'epoch': 0.1} +{'loss': 2.4803, 'grad_norm': 2.5526316165924072, 'learning_rate': 1.4233576642335767e-06, 'epoch': 0.1} +{'loss': 2.5149, 'grad_norm': 2.4569883346557617, 'learning_rate': 1.4416058394160583e-06, 'epoch': 0.1} +{'loss': 2.4616, 'grad_norm': 2.9641432762145996, 'learning_rate': 1.4598540145985402e-06, 'epoch': 0.1} +{'loss': 2.429, 'grad_norm': 2.821410894393921, 'learning_rate': 1.478102189781022e-06, 'epoch': 0.1} +{'loss': 2.5028, 'grad_norm': 3.8620495796203613, 'learning_rate': 1.496350364963504e-06, 'epoch': 0.11} +{'loss': 2.4631, 'grad_norm': 3.0422098636627197, 'learning_rate': 1.5145985401459856e-06, 'epoch': 0.11} +{'loss': 2.4786, 'grad_norm': 2.8717942237854004, 'learning_rate': 1.5328467153284674e-06, 'epoch': 0.11} +{'loss': 2.4331, 'grad_norm': 2.6856589317321777, 'learning_rate': 1.551094890510949e-06, 'epoch': 0.11} +{'loss': 2.3742, 'grad_norm': 4.284306526184082, 'learning_rate': 1.5693430656934307e-06, 'epoch': 0.11} +{'eval_loss': 2.514993667602539, 'eval_runtime': 37.0907, 'eval_samples_per_second': 26.961, 'eval_steps_per_second': 1.132, 'epoch': 0.11} +{'loss': 2.3809, 'grad_norm': 4.973050117492676, 'learning_rate': 1.5875912408759126e-06, 'epoch': 0.11} +{'loss': 2.4314, 'grad_norm': 2.808378219604492, 'learning_rate': 1.6058394160583942e-06, 'epoch': 0.11} +{'loss': 2.4848, 'grad_norm': 3.9805150032043457, 'learning_rate': 1.624087591240876e-06, 'epoch': 0.11} +{'loss': 2.3981, 'grad_norm': 2.7861433029174805, 'learning_rate': 1.6423357664233577e-06, 'epoch': 0.11} +{'loss': 2.3422, 'grad_norm': 3.4071974754333496, 'learning_rate': 1.6605839416058394e-06, 'epoch': 0.12} +{'loss': 2.3764, 'grad_norm': 2.4880754947662354, 'learning_rate': 1.6788321167883212e-06, 'epoch': 0.12} +{'loss': 2.3969, 'grad_norm': 3.5724127292633057, 'learning_rate': 1.6970802919708029e-06, 'epoch': 0.12} +{'loss': 2.3188, 'grad_norm': 2.5322089195251465, 'learning_rate': 1.715328467153285e-06, 'epoch': 0.12} +{'loss': 2.3672, 'grad_norm': 3.194002628326416, 'learning_rate': 1.7335766423357666e-06, 'epoch': 0.12} +{'loss': 2.3749, 'grad_norm': 2.489720344543457, 'learning_rate': 1.7518248175182485e-06, 'epoch': 0.12} +{'eval_loss': 2.455575704574585, 'eval_runtime': 36.8723, 'eval_samples_per_second': 27.121, 'eval_steps_per_second': 1.139, 'epoch': 0.12} +{'loss': 2.4069, 'grad_norm': 2.7961671352386475, 'learning_rate': 1.77007299270073e-06, 'epoch': 0.12} +{'loss': 2.3764, 'grad_norm': 2.6191494464874268, 'learning_rate': 1.788321167883212e-06, 'epoch': 0.12} +{'loss': 2.4182, 'grad_norm': 2.5566232204437256, 'learning_rate': 1.8065693430656936e-06, 'epoch': 0.12} +{'loss': 2.3665, 'grad_norm': 2.5462262630462646, 'learning_rate': 1.8248175182481753e-06, 'epoch': 0.12} +{'loss': 2.3045, 'grad_norm': 2.6968109607696533, 'learning_rate': 1.8430656934306571e-06, 'epoch': 0.13} +{'loss': 2.3332, 'grad_norm': 2.838625192642212, 'learning_rate': 1.8613138686131388e-06, 'epoch': 0.13} +{'loss': 2.3204, 'grad_norm': 2.584407091140747, 'learning_rate': 1.8795620437956206e-06, 'epoch': 0.13} +{'loss': 2.312, 'grad_norm': 2.5297932624816895, 'learning_rate': 1.8978102189781023e-06, 'epoch': 0.13} +{'loss': 2.3271, 'grad_norm': 2.6709144115448, 'learning_rate': 1.916058394160584e-06, 'epoch': 0.13} +{'loss': 2.3415, 'grad_norm': 2.6457643508911133, 'learning_rate': 1.934306569343066e-06, 'epoch': 0.13} +{'eval_loss': 2.4165854454040527, 'eval_runtime': 36.7731, 'eval_samples_per_second': 27.194, 'eval_steps_per_second': 1.142, 'epoch': 0.13} +{'loss': 2.2405, 'grad_norm': 3.8355023860931396, 'learning_rate': 1.952554744525548e-06, 'epoch': 0.13} +{'loss': 2.2841, 'grad_norm': 2.6029245853424072, 'learning_rate': 1.9708029197080293e-06, 'epoch': 0.13} +{'loss': 2.2975, 'grad_norm': 3.9318411350250244, 'learning_rate': 1.989051094890511e-06, 'epoch': 0.13} +{'loss': 2.2963, 'grad_norm': 4.0954461097717285, 'learning_rate': 2.007299270072993e-06, 'epoch': 0.14} +{'loss': 2.3526, 'grad_norm': 2.4977500438690186, 'learning_rate': 2.025547445255475e-06, 'epoch': 0.14} +{'loss': 2.2856, 'grad_norm': 4.252346992492676, 'learning_rate': 2.0437956204379563e-06, 'epoch': 0.14} +{'loss': 2.294, 'grad_norm': 2.8565123081207275, 'learning_rate': 2.062043795620438e-06, 'epoch': 0.14} +{'loss': 2.2817, 'grad_norm': 3.1502203941345215, 'learning_rate': 2.08029197080292e-06, 'epoch': 0.14} +{'loss': 2.3483, 'grad_norm': 3.80749249458313, 'learning_rate': 2.0985401459854014e-06, 'epoch': 0.14} +{'loss': 2.2368, 'grad_norm': 3.1611809730529785, 'learning_rate': 2.1167883211678833e-06, 'epoch': 0.14} +{'eval_loss': 2.3837099075317383, 'eval_runtime': 36.8798, 'eval_samples_per_second': 27.115, 'eval_steps_per_second': 1.139, 'epoch': 0.14} +{'loss': 2.3034, 'grad_norm': 3.130007028579712, 'learning_rate': 2.135036496350365e-06, 'epoch': 0.14} +{'loss': 2.3314, 'grad_norm': 3.0841190814971924, 'learning_rate': 2.1532846715328466e-06, 'epoch': 0.14} +{'loss': 2.2322, 'grad_norm': 2.8934333324432373, 'learning_rate': 2.171532846715329e-06, 'epoch': 0.15} +{'loss': 2.2449, 'grad_norm': 2.8835387229919434, 'learning_rate': 2.1897810218978103e-06, 'epoch': 0.15} +{'loss': 2.2861, 'grad_norm': 3.428131103515625, 'learning_rate': 2.208029197080292e-06, 'epoch': 0.15} +{'loss': 2.2611, 'grad_norm': 2.794792652130127, 'learning_rate': 2.226277372262774e-06, 'epoch': 0.15} +{'loss': 2.2668, 'grad_norm': 3.3428287506103516, 'learning_rate': 2.244525547445256e-06, 'epoch': 0.15} +{'loss': 2.2943, 'grad_norm': 3.4636261463165283, 'learning_rate': 2.2627737226277373e-06, 'epoch': 0.15} +{'loss': 2.2869, 'grad_norm': 2.7653632164001465, 'learning_rate': 2.281021897810219e-06, 'epoch': 0.15} +{'loss': 2.2447, 'grad_norm': 3.011944532394409, 'learning_rate': 2.299270072992701e-06, 'epoch': 0.15} +{'eval_loss': 2.360229969024658, 'eval_runtime': 36.8305, 'eval_samples_per_second': 27.151, 'eval_steps_per_second': 1.14, 'epoch': 0.15} +{'loss': 2.2391, 'grad_norm': 3.198716640472412, 'learning_rate': 2.3175182481751825e-06, 'epoch': 0.15} +{'loss': 2.2547, 'grad_norm': 2.816667079925537, 'learning_rate': 2.3357664233576643e-06, 'epoch': 0.16} +{'loss': 2.2036, 'grad_norm': 2.6860835552215576, 'learning_rate': 2.354014598540146e-06, 'epoch': 0.16} +{'loss': 2.2461, 'grad_norm': 2.654677152633667, 'learning_rate': 2.3722627737226276e-06, 'epoch': 0.16} +{'loss': 2.2565, 'grad_norm': 2.617124080657959, 'learning_rate': 2.39051094890511e-06, 'epoch': 0.16} +{'loss': 2.1914, 'grad_norm': 3.1751718521118164, 'learning_rate': 2.4087591240875918e-06, 'epoch': 0.16} +{'loss': 2.2179, 'grad_norm': 2.6073708534240723, 'learning_rate': 2.427007299270073e-06, 'epoch': 0.16} +{'loss': 2.2254, 'grad_norm': 3.581951141357422, 'learning_rate': 2.445255474452555e-06, 'epoch': 0.16} +{'loss': 2.2691, 'grad_norm': 3.2456817626953125, 'learning_rate': 2.463503649635037e-06, 'epoch': 0.16} +{'loss': 2.1837, 'grad_norm': 3.040400505065918, 'learning_rate': 2.4817518248175183e-06, 'epoch': 0.16} +{'eval_loss': 2.3398873805999756, 'eval_runtime': 36.9778, 'eval_samples_per_second': 27.043, 'eval_steps_per_second': 1.136, 'epoch': 0.16} +{'loss': 2.1501, 'grad_norm': 3.313336133956909, 'learning_rate': 2.5e-06, 'epoch': 0.17} +{'loss': 2.2836, 'grad_norm': 3.2824361324310303, 'learning_rate': 2.518248175182482e-06, 'epoch': 0.17} +{'loss': 2.1755, 'grad_norm': 2.8979086875915527, 'learning_rate': 2.5364963503649635e-06, 'epoch': 0.17} +{'loss': 2.1791, 'grad_norm': 2.945603609085083, 'learning_rate': 2.5547445255474458e-06, 'epoch': 0.17} +{'loss': 2.2338, 'grad_norm': 2.6949260234832764, 'learning_rate': 2.5729927007299272e-06, 'epoch': 0.17} +{'loss': 2.2067, 'grad_norm': 3.3816075325012207, 'learning_rate': 2.591240875912409e-06, 'epoch': 0.17} +{'loss': 2.2064, 'grad_norm': 2.516245126724243, 'learning_rate': 2.6094890510948905e-06, 'epoch': 0.17} +{'loss': 2.2417, 'grad_norm': 3.5714497566223145, 'learning_rate': 2.627737226277373e-06, 'epoch': 0.17} +{'loss': 2.244, 'grad_norm': 2.8084025382995605, 'learning_rate': 2.6459854014598542e-06, 'epoch': 0.17} +{'loss': 2.2036, 'grad_norm': 2.94307541847229, 'learning_rate': 2.664233576642336e-06, 'epoch': 0.18} +{'eval_loss': 2.3238048553466797, 'eval_runtime': 36.8827, 'eval_samples_per_second': 27.113, 'eval_steps_per_second': 1.139, 'epoch': 0.18} +{'loss': 2.1825, 'grad_norm': 3.5338687896728516, 'learning_rate': 2.6824817518248175e-06, 'epoch': 0.18} +{'loss': 2.1991, 'grad_norm': 2.6816883087158203, 'learning_rate': 2.7007299270072994e-06, 'epoch': 0.18} +{'loss': 2.1771, 'grad_norm': 2.9373624324798584, 'learning_rate': 2.718978102189781e-06, 'epoch': 0.18} +{'loss': 2.2348, 'grad_norm': 3.060258388519287, 'learning_rate': 2.737226277372263e-06, 'epoch': 0.18} +{'loss': 2.2295, 'grad_norm': 2.9374494552612305, 'learning_rate': 2.755474452554745e-06, 'epoch': 0.18} +{'loss': 2.1743, 'grad_norm': 2.9218924045562744, 'learning_rate': 2.7737226277372264e-06, 'epoch': 0.18} +{'loss': 2.2821, 'grad_norm': 2.7671658992767334, 'learning_rate': 2.7919708029197082e-06, 'epoch': 0.18} +{'loss': 2.2369, 'grad_norm': 2.830994129180908, 'learning_rate': 2.81021897810219e-06, 'epoch': 0.18} +{'loss': 2.1718, 'grad_norm': 2.4539380073547363, 'learning_rate': 2.828467153284672e-06, 'epoch': 0.19} +{'loss': 2.2034, 'grad_norm': 2.529789686203003, 'learning_rate': 2.8467153284671534e-06, 'epoch': 0.19} +{'eval_loss': 2.3094632625579834, 'eval_runtime': 36.9061, 'eval_samples_per_second': 27.096, 'eval_steps_per_second': 1.138, 'epoch': 0.19} +{'loss': 2.1205, 'grad_norm': 2.776603937149048, 'learning_rate': 2.8649635036496353e-06, 'epoch': 0.19} +{'loss': 2.1788, 'grad_norm': 2.6126351356506348, 'learning_rate': 2.8832116788321167e-06, 'epoch': 0.19} +{'loss': 2.1306, 'grad_norm': 2.5450496673583984, 'learning_rate': 2.901459854014599e-06, 'epoch': 0.19} +{'loss': 2.2277, 'grad_norm': 2.5038654804229736, 'learning_rate': 2.9197080291970804e-06, 'epoch': 0.19} +{'loss': 2.2078, 'grad_norm': 2.7198233604431152, 'learning_rate': 2.9379562043795623e-06, 'epoch': 0.19} +{'loss': 2.1902, 'grad_norm': 3.0290682315826416, 'learning_rate': 2.956204379562044e-06, 'epoch': 0.19} +{'loss': 2.236, 'grad_norm': 2.661337375640869, 'learning_rate': 2.9744525547445256e-06, 'epoch': 0.19} +{'loss': 2.1875, 'grad_norm': 2.7548766136169434, 'learning_rate': 2.992700729927008e-06, 'epoch': 0.2} +{'loss': 2.1447, 'grad_norm': 3.7520737648010254, 'learning_rate': 3.0109489051094893e-06, 'epoch': 0.2} +{'loss': 2.1503, 'grad_norm': 3.4713258743286133, 'learning_rate': 3.029197080291971e-06, 'epoch': 0.2} +{'eval_loss': 2.294600486755371, 'eval_runtime': 36.8803, 'eval_samples_per_second': 27.115, 'eval_steps_per_second': 1.139, 'epoch': 0.2} +{'loss': 2.2199, 'grad_norm': 2.3800981044769287, 'learning_rate': 3.0474452554744526e-06, 'epoch': 0.2} +{'loss': 2.2165, 'grad_norm': 2.623415946960449, 'learning_rate': 3.065693430656935e-06, 'epoch': 0.2} +{'loss': 2.1962, 'grad_norm': 2.571760416030884, 'learning_rate': 3.0839416058394163e-06, 'epoch': 0.2} +{'loss': 2.2164, 'grad_norm': 2.6511805057525635, 'learning_rate': 3.102189781021898e-06, 'epoch': 0.2} +{'loss': 2.1988, 'grad_norm': 2.8188459873199463, 'learning_rate': 3.1204379562043796e-06, 'epoch': 0.2} +{'loss': 2.1246, 'grad_norm': 2.6241681575775146, 'learning_rate': 3.1386861313868614e-06, 'epoch': 0.2} +{'loss': 2.1162, 'grad_norm': 2.674527645111084, 'learning_rate': 3.156934306569343e-06, 'epoch': 0.2} +{'loss': 2.2266, 'grad_norm': 2.8179783821105957, 'learning_rate': 3.175182481751825e-06, 'epoch': 0.21} +{'loss': 2.1767, 'grad_norm': 2.5623691082000732, 'learning_rate': 3.193430656934307e-06, 'epoch': 0.21} +{'loss': 2.1556, 'grad_norm': 2.6922426223754883, 'learning_rate': 3.2116788321167884e-06, 'epoch': 0.21} +{'eval_loss': 2.284255266189575, 'eval_runtime': 37.1234, 'eval_samples_per_second': 26.937, 'eval_steps_per_second': 1.131, 'epoch': 0.21} +{'loss': 2.1701, 'grad_norm': 2.670802116394043, 'learning_rate': 3.2299270072992707e-06, 'epoch': 0.21} +{'loss': 2.1563, 'grad_norm': 2.7782397270202637, 'learning_rate': 3.248175182481752e-06, 'epoch': 0.21} +{'loss': 2.0815, 'grad_norm': 2.993387460708618, 'learning_rate': 3.266423357664234e-06, 'epoch': 0.21} +{'loss': 2.1791, 'grad_norm': 2.964684009552002, 'learning_rate': 3.2846715328467155e-06, 'epoch': 0.21} +{'loss': 2.1535, 'grad_norm': 2.738447427749634, 'learning_rate': 3.3029197080291973e-06, 'epoch': 0.21} +{'loss': 2.1879, 'grad_norm': 3.247713804244995, 'learning_rate': 3.3211678832116788e-06, 'epoch': 0.21} +{'loss': 2.1647, 'grad_norm': 3.095205783843994, 'learning_rate': 3.339416058394161e-06, 'epoch': 0.22} +{'loss': 2.1583, 'grad_norm': 2.6435153484344482, 'learning_rate': 3.3576642335766425e-06, 'epoch': 0.22} +{'loss': 2.1569, 'grad_norm': 3.1337387561798096, 'learning_rate': 3.3759124087591243e-06, 'epoch': 0.22} +{'loss': 2.1633, 'grad_norm': 3.172367572784424, 'learning_rate': 3.3941605839416058e-06, 'epoch': 0.22} +{'eval_loss': 2.270273208618164, 'eval_runtime': 37.1183, 'eval_samples_per_second': 26.941, 'eval_steps_per_second': 1.132, 'epoch': 0.22} +{'loss': 2.1633, 'grad_norm': 2.533245801925659, 'learning_rate': 3.412408759124088e-06, 'epoch': 0.22} +{'loss': 2.1687, 'grad_norm': 3.0553905963897705, 'learning_rate': 3.43065693430657e-06, 'epoch': 0.22} +{'loss': 2.1064, 'grad_norm': 3.2229137420654297, 'learning_rate': 3.4489051094890513e-06, 'epoch': 0.22} +{'loss': 2.1584, 'grad_norm': 2.752973794937134, 'learning_rate': 3.467153284671533e-06, 'epoch': 0.22} +{'loss': 2.1016, 'grad_norm': 2.75563645362854, 'learning_rate': 3.4854014598540146e-06, 'epoch': 0.22} +{'loss': 2.143, 'grad_norm': 2.480123281478882, 'learning_rate': 3.503649635036497e-06, 'epoch': 0.23} +{'loss': 2.1388, 'grad_norm': 2.7256920337677, 'learning_rate': 3.5218978102189783e-06, 'epoch': 0.23} +{'loss': 2.1712, 'grad_norm': 2.680922746658325, 'learning_rate': 3.54014598540146e-06, 'epoch': 0.23} +{'loss': 2.1807, 'grad_norm': 3.1483004093170166, 'learning_rate': 3.5583941605839416e-06, 'epoch': 0.23} +{'loss': 2.23, 'grad_norm': 2.7536072731018066, 'learning_rate': 3.576642335766424e-06, 'epoch': 0.23} +{'eval_loss': 2.264357805252075, 'eval_runtime': 36.9285, 'eval_samples_per_second': 27.079, 'eval_steps_per_second': 1.137, 'epoch': 0.23} +{'loss': 2.211, 'grad_norm': 2.9679646492004395, 'learning_rate': 3.5948905109489054e-06, 'epoch': 0.23} +{'loss': 2.0679, 'grad_norm': 2.755133628845215, 'learning_rate': 3.6131386861313872e-06, 'epoch': 0.23} +{'loss': 2.064, 'grad_norm': 3.1508214473724365, 'learning_rate': 3.6313868613138687e-06, 'epoch': 0.23} +{'loss': 2.0942, 'grad_norm': 3.004370927810669, 'learning_rate': 3.6496350364963505e-06, 'epoch': 0.23} +{'loss': 2.1168, 'grad_norm': 2.5298097133636475, 'learning_rate': 3.667883211678833e-06, 'epoch': 0.24} +{'loss': 2.0983, 'grad_norm': 3.2273404598236084, 'learning_rate': 3.6861313868613142e-06, 'epoch': 0.24} +{'loss': 2.0932, 'grad_norm': 2.6995913982391357, 'learning_rate': 3.704379562043796e-06, 'epoch': 0.24} +{'loss': 2.1848, 'grad_norm': 2.9947187900543213, 'learning_rate': 3.7226277372262775e-06, 'epoch': 0.24} +{'loss': 2.1533, 'grad_norm': 2.8766634464263916, 'learning_rate': 3.7408759124087594e-06, 'epoch': 0.24} +{'loss': 2.0936, 'grad_norm': 3.011009454727173, 'learning_rate': 3.7591240875912412e-06, 'epoch': 0.24} +{'eval_loss': 2.250859260559082, 'eval_runtime': 36.8414, 'eval_samples_per_second': 27.143, 'eval_steps_per_second': 1.14, 'epoch': 0.24} +{'loss': 2.1338, 'grad_norm': 2.5399389266967773, 'learning_rate': 3.777372262773723e-06, 'epoch': 0.24} +{'loss': 2.1216, 'grad_norm': 2.793663263320923, 'learning_rate': 3.7956204379562045e-06, 'epoch': 0.24} +{'loss': 2.0957, 'grad_norm': 2.8745079040527344, 'learning_rate': 3.8138686131386864e-06, 'epoch': 0.24} +{'loss': 2.1489, 'grad_norm': 2.521819591522217, 'learning_rate': 3.832116788321168e-06, 'epoch': 0.25} +{'loss': 2.11, 'grad_norm': 2.724716901779175, 'learning_rate': 3.85036496350365e-06, 'epoch': 0.25} +{'loss': 2.0759, 'grad_norm': 2.4324700832366943, 'learning_rate': 3.868613138686132e-06, 'epoch': 0.25} +{'loss': 2.1131, 'grad_norm': 3.0802838802337646, 'learning_rate': 3.886861313868613e-06, 'epoch': 0.25} +{'loss': 2.125, 'grad_norm': 2.79115629196167, 'learning_rate': 3.905109489051096e-06, 'epoch': 0.25} +{'loss': 2.1606, 'grad_norm': 2.475189685821533, 'learning_rate': 3.923357664233577e-06, 'epoch': 0.25} +{'loss': 2.1165, 'grad_norm': 2.7638697624206543, 'learning_rate': 3.9416058394160585e-06, 'epoch': 0.25} +{'eval_loss': 2.2373578548431396, 'eval_runtime': 36.8357, 'eval_samples_per_second': 27.148, 'eval_steps_per_second': 1.14, 'epoch': 0.25} +{'loss': 2.1662, 'grad_norm': 2.5692026615142822, 'learning_rate': 3.95985401459854e-06, 'epoch': 0.25} +{'loss': 2.1864, 'grad_norm': 2.398109197616577, 'learning_rate': 3.978102189781022e-06, 'epoch': 0.25} +{'loss': 2.0537, 'grad_norm': 2.628312826156616, 'learning_rate': 3.996350364963504e-06, 'epoch': 0.26} +{'loss': 2.1348, 'grad_norm': 2.615253448486328, 'learning_rate': 4.014598540145986e-06, 'epoch': 0.26} +{'loss': 2.0396, 'grad_norm': 2.499142646789551, 'learning_rate': 4.032846715328467e-06, 'epoch': 0.26} +{'loss': 2.0969, 'grad_norm': 2.7101237773895264, 'learning_rate': 4.05109489051095e-06, 'epoch': 0.26} +{'loss': 2.1229, 'grad_norm': 2.574430227279663, 'learning_rate': 4.069343065693431e-06, 'epoch': 0.26} +{'loss': 2.0845, 'grad_norm': 2.6702752113342285, 'learning_rate': 4.0875912408759126e-06, 'epoch': 0.26} +{'loss': 2.1096, 'grad_norm': 3.2056877613067627, 'learning_rate': 4.105839416058395e-06, 'epoch': 0.26} +{'loss': 2.1783, 'grad_norm': 2.633591413497925, 'learning_rate': 4.124087591240876e-06, 'epoch': 0.26} +{'eval_loss': 2.2297379970550537, 'eval_runtime': 36.8474, 'eval_samples_per_second': 27.139, 'eval_steps_per_second': 1.14, 'epoch': 0.26} +{'loss': 2.0974, 'grad_norm': 2.780073642730713, 'learning_rate': 4.1423357664233586e-06, 'epoch': 0.26} +{'loss': 2.0772, 'grad_norm': 2.7121286392211914, 'learning_rate': 4.16058394160584e-06, 'epoch': 0.27} +{'loss': 2.1567, 'grad_norm': 3.238070011138916, 'learning_rate': 4.1788321167883214e-06, 'epoch': 0.27} +{'loss': 2.0467, 'grad_norm': 3.9809019565582275, 'learning_rate': 4.197080291970803e-06, 'epoch': 0.27} +{'loss': 2.1199, 'grad_norm': 2.440645217895508, 'learning_rate': 4.215328467153285e-06, 'epoch': 0.27} +{'loss': 2.0787, 'grad_norm': 3.84128999710083, 'learning_rate': 4.233576642335767e-06, 'epoch': 0.27} +{'loss': 2.0756, 'grad_norm': 2.7698514461517334, 'learning_rate': 4.251824817518249e-06, 'epoch': 0.27} +{'loss': 2.1093, 'grad_norm': 2.625366687774658, 'learning_rate': 4.27007299270073e-06, 'epoch': 0.27} +{'loss': 2.0747, 'grad_norm': 2.6527903079986572, 'learning_rate': 4.288321167883212e-06, 'epoch': 0.27} +{'loss': 2.0708, 'grad_norm': 2.7589945793151855, 'learning_rate': 4.306569343065693e-06, 'epoch': 0.27} +{'eval_loss': 2.221503734588623, 'eval_runtime': 36.932, 'eval_samples_per_second': 27.077, 'eval_steps_per_second': 1.137, 'epoch': 0.27} +{'loss': 2.0421, 'grad_norm': 2.598238468170166, 'learning_rate': 4.3248175182481755e-06, 'epoch': 0.28} +{'loss': 2.0997, 'grad_norm': 2.893688440322876, 'learning_rate': 4.343065693430658e-06, 'epoch': 0.28} +{'loss': 2.0941, 'grad_norm': 2.436887741088867, 'learning_rate': 4.361313868613139e-06, 'epoch': 0.28} +{'loss': 2.0818, 'grad_norm': 2.671720504760742, 'learning_rate': 4.379562043795621e-06, 'epoch': 0.28} +{'loss': 2.1164, 'grad_norm': 2.575180768966675, 'learning_rate': 4.397810218978102e-06, 'epoch': 0.28} +{'loss': 2.1104, 'grad_norm': 2.7255051136016846, 'learning_rate': 4.416058394160584e-06, 'epoch': 0.28} +{'loss': 2.0887, 'grad_norm': 2.75199031829834, 'learning_rate': 4.434306569343066e-06, 'epoch': 0.28} +{'loss': 2.0612, 'grad_norm': 2.5259952545166016, 'learning_rate': 4.452554744525548e-06, 'epoch': 0.28} +{'loss': 2.1114, 'grad_norm': 3.1171822547912598, 'learning_rate': 4.4708029197080295e-06, 'epoch': 0.28} +{'loss': 2.1016, 'grad_norm': 2.5662221908569336, 'learning_rate': 4.489051094890512e-06, 'epoch': 0.28} +{'eval_loss': 2.2163822650909424, 'eval_runtime': 36.8437, 'eval_samples_per_second': 27.142, 'eval_steps_per_second': 1.14, 'epoch': 0.28} +{'loss': 2.1404, 'grad_norm': 2.3760831356048584, 'learning_rate': 4.507299270072993e-06, 'epoch': 0.29} +{'loss': 2.1037, 'grad_norm': 3.341439723968506, 'learning_rate': 4.525547445255475e-06, 'epoch': 0.29} +{'loss': 2.0863, 'grad_norm': 2.7790281772613525, 'learning_rate': 4.543795620437956e-06, 'epoch': 0.29} +{'loss': 2.0555, 'grad_norm': 2.9859230518341064, 'learning_rate': 4.562043795620438e-06, 'epoch': 0.29} +{'loss': 2.1464, 'grad_norm': 2.668275833129883, 'learning_rate': 4.580291970802921e-06, 'epoch': 0.29} +{'loss': 2.1494, 'grad_norm': 2.858091115951538, 'learning_rate': 4.598540145985402e-06, 'epoch': 0.29} +{'loss': 2.109, 'grad_norm': 2.5387940406799316, 'learning_rate': 4.6167883211678835e-06, 'epoch': 0.29} +{'loss': 2.0626, 'grad_norm': 2.4841413497924805, 'learning_rate': 4.635036496350365e-06, 'epoch': 0.29} +{'loss': 2.0635, 'grad_norm': 2.696976661682129, 'learning_rate': 4.653284671532847e-06, 'epoch': 0.29} +{'loss': 2.1056, 'grad_norm': 2.710571527481079, 'learning_rate': 4.671532846715329e-06, 'epoch': 0.3} +{'eval_loss': 2.209902286529541, 'eval_runtime': 36.8342, 'eval_samples_per_second': 27.149, 'eval_steps_per_second': 1.14, 'epoch': 0.3} +{'loss': 2.1186, 'grad_norm': 2.6863739490509033, 'learning_rate': 4.689781021897811e-06, 'epoch': 0.3} +{'loss': 2.0856, 'grad_norm': 2.546550989151001, 'learning_rate': 4.708029197080292e-06, 'epoch': 0.3} +{'loss': 2.1301, 'grad_norm': 2.949937582015991, 'learning_rate': 4.726277372262774e-06, 'epoch': 0.3} +{'loss': 2.0987, 'grad_norm': 2.292760133743286, 'learning_rate': 4.744525547445255e-06, 'epoch': 0.3} +{'loss': 2.0296, 'grad_norm': 2.5523879528045654, 'learning_rate': 4.7627737226277375e-06, 'epoch': 0.3} +{'loss': 2.1018, 'grad_norm': 2.2079036235809326, 'learning_rate': 4.78102189781022e-06, 'epoch': 0.3} +{'loss': 2.0953, 'grad_norm': 2.5277910232543945, 'learning_rate': 4.799270072992701e-06, 'epoch': 0.3} +{'loss': 2.0709, 'grad_norm': 2.5521962642669678, 'learning_rate': 4.8175182481751835e-06, 'epoch': 0.3} +{'loss': 2.0961, 'grad_norm': 2.2348296642303467, 'learning_rate': 4.835766423357665e-06, 'epoch': 0.31} +{'loss': 2.0775, 'grad_norm': 2.782870292663574, 'learning_rate': 4.854014598540146e-06, 'epoch': 0.31} +{'eval_loss': 2.2114169597625732, 'eval_runtime': 36.9089, 'eval_samples_per_second': 27.094, 'eval_steps_per_second': 1.138, 'epoch': 0.31} +{'loss': 2.03, 'grad_norm': 2.6238420009613037, 'learning_rate': 4.872262773722628e-06, 'epoch': 0.31} +{'loss': 2.0434, 'grad_norm': 2.29125714302063, 'learning_rate': 4.89051094890511e-06, 'epoch': 0.31} +{'loss': 2.0284, 'grad_norm': 2.3889894485473633, 'learning_rate': 4.9087591240875915e-06, 'epoch': 0.31} +{'loss': 2.0695, 'grad_norm': 2.3177151679992676, 'learning_rate': 4.927007299270074e-06, 'epoch': 0.31} +{'loss': 2.0729, 'grad_norm': 2.437307596206665, 'learning_rate': 4.945255474452555e-06, 'epoch': 0.31} +{'loss': 2.0463, 'grad_norm': 2.3625094890594482, 'learning_rate': 4.963503649635037e-06, 'epoch': 0.31} +{'loss': 2.1376, 'grad_norm': 2.32767391204834, 'learning_rate': 4.981751824817518e-06, 'epoch': 0.31} +{'loss': 2.132, 'grad_norm': 2.291435956954956, 'learning_rate': 5e-06, 'epoch': 0.32} +{'loss': 2.0759, 'grad_norm': 2.226759910583496, 'learning_rate': 4.997969130787978e-06, 'epoch': 0.32} +{'loss': 2.1042, 'grad_norm': 2.2493679523468018, 'learning_rate': 4.995938261575955e-06, 'epoch': 0.32} +{'eval_loss': 2.1926615238189697, 'eval_runtime': 36.9552, 'eval_samples_per_second': 27.06, 'eval_steps_per_second': 1.137, 'epoch': 0.32} +{'loss': 2.012, 'grad_norm': 2.4757614135742188, 'learning_rate': 4.993907392363932e-06, 'epoch': 0.32} +{'loss': 2.0936, 'grad_norm': 2.420912981033325, 'learning_rate': 4.9918765231519095e-06, 'epoch': 0.32} +{'loss': 2.0119, 'grad_norm': 2.3863155841827393, 'learning_rate': 4.989845653939887e-06, 'epoch': 0.32} +{'loss': 2.0625, 'grad_norm': 2.4652020931243896, 'learning_rate': 4.987814784727864e-06, 'epoch': 0.32} +{'loss': 2.0591, 'grad_norm': 2.2076728343963623, 'learning_rate': 4.985783915515841e-06, 'epoch': 0.32} +{'loss': 2.0548, 'grad_norm': 2.4880995750427246, 'learning_rate': 4.983753046303819e-06, 'epoch': 0.32} +{'loss': 2.0678, 'grad_norm': 2.396613359451294, 'learning_rate': 4.981722177091796e-06, 'epoch': 0.33} +{'loss': 2.0594, 'grad_norm': 2.6646151542663574, 'learning_rate': 4.979691307879773e-06, 'epoch': 0.33} +{'loss': 2.0862, 'grad_norm': 2.393214464187622, 'learning_rate': 4.9776604386677504e-06, 'epoch': 0.33} +{'loss': 2.0659, 'grad_norm': 2.651883840560913, 'learning_rate': 4.975629569455728e-06, 'epoch': 0.33} +{'eval_loss': 2.1894888877868652, 'eval_runtime': 37.0477, 'eval_samples_per_second': 26.992, 'eval_steps_per_second': 1.134, 'epoch': 0.33} +{'loss': 2.1162, 'grad_norm': 2.3431386947631836, 'learning_rate': 4.973598700243705e-06, 'epoch': 0.33} +{'loss': 2.0276, 'grad_norm': 2.754340171813965, 'learning_rate': 4.971567831031681e-06, 'epoch': 0.33} +{'loss': 2.038, 'grad_norm': 3.2804505825042725, 'learning_rate': 4.9695369618196595e-06, 'epoch': 0.33} +{'loss': 1.9115, 'grad_norm': 2.722909927368164, 'learning_rate': 4.967506092607636e-06, 'epoch': 0.33} +{'loss': 2.0034, 'grad_norm': 2.82663893699646, 'learning_rate': 4.965475223395614e-06, 'epoch': 0.33} +{'loss': 2.0251, 'grad_norm': 2.743824005126953, 'learning_rate': 4.9634443541835905e-06, 'epoch': 0.34} +{'loss': 2.0359, 'grad_norm': 2.371054172515869, 'learning_rate': 4.961413484971569e-06, 'epoch': 0.34} +{'loss': 2.0489, 'grad_norm': 2.5377180576324463, 'learning_rate': 4.959382615759545e-06, 'epoch': 0.34} +{'loss': 2.0567, 'grad_norm': 2.7547554969787598, 'learning_rate': 4.957351746547523e-06, 'epoch': 0.34} +{'loss': 2.032, 'grad_norm': 2.832305669784546, 'learning_rate': 4.9553208773355e-06, 'epoch': 0.34} +{'eval_loss': 2.19657301902771, 'eval_runtime': 36.8259, 'eval_samples_per_second': 27.155, 'eval_steps_per_second': 1.141, 'epoch': 0.34} +{'loss': 2.1559, 'grad_norm': 2.46624493598938, 'learning_rate': 4.953290008123478e-06, 'epoch': 0.34} +{'loss': 2.0618, 'grad_norm': 2.380703926086426, 'learning_rate': 4.951259138911454e-06, 'epoch': 0.34} +{'loss': 1.9843, 'grad_norm': 2.3415584564208984, 'learning_rate': 4.9492282696994315e-06, 'epoch': 0.34} +{'loss': 2.1236, 'grad_norm': 2.4721949100494385, 'learning_rate': 4.947197400487409e-06, 'epoch': 0.34} +{'loss': 2.0454, 'grad_norm': 2.5910539627075195, 'learning_rate': 4.945166531275386e-06, 'epoch': 0.35} +{'loss': 2.0419, 'grad_norm': 2.761718511581421, 'learning_rate': 4.943135662063363e-06, 'epoch': 0.35} +{'loss': 2.0471, 'grad_norm': 3.101546287536621, 'learning_rate': 4.9411047928513406e-06, 'epoch': 0.35} +{'loss': 2.0834, 'grad_norm': 2.7173798084259033, 'learning_rate': 4.939073923639318e-06, 'epoch': 0.35} +{'loss': 2.0889, 'grad_norm': 2.716454267501831, 'learning_rate': 4.937043054427295e-06, 'epoch': 0.35} +{'loss': 2.0245, 'grad_norm': 2.755162239074707, 'learning_rate': 4.935012185215272e-06, 'epoch': 0.35} +{'eval_loss': 2.1819753646850586, 'eval_runtime': 36.9094, 'eval_samples_per_second': 27.093, 'eval_steps_per_second': 1.138, 'epoch': 0.35} +{'loss': 2.0095, 'grad_norm': 2.6325459480285645, 'learning_rate': 4.93298131600325e-06, 'epoch': 0.35} +{'loss': 2.0745, 'grad_norm': 2.909247398376465, 'learning_rate': 4.930950446791227e-06, 'epoch': 0.35} +{'loss': 2.0531, 'grad_norm': 3.1001574993133545, 'learning_rate': 4.928919577579204e-06, 'epoch': 0.35} +{'loss': 2.0562, 'grad_norm': 2.546571731567383, 'learning_rate': 4.9268887083671815e-06, 'epoch': 0.36} +{'loss': 2.0396, 'grad_norm': 3.4734079837799072, 'learning_rate': 4.924857839155159e-06, 'epoch': 0.36} +{'loss': 2.0418, 'grad_norm': 2.253105878829956, 'learning_rate': 4.922826969943136e-06, 'epoch': 0.36} +{'loss': 2.0683, 'grad_norm': 3.5297048091888428, 'learning_rate': 4.920796100731113e-06, 'epoch': 0.36} +{'loss': 1.9887, 'grad_norm': 2.448322296142578, 'learning_rate': 4.918765231519091e-06, 'epoch': 0.36} +{'loss': 2.0489, 'grad_norm': 2.4430301189422607, 'learning_rate': 4.916734362307068e-06, 'epoch': 0.36} +{'loss': 1.9955, 'grad_norm': 2.4111526012420654, 'learning_rate': 4.914703493095045e-06, 'epoch': 0.36} +{'eval_loss': 2.1826932430267334, 'eval_runtime': 36.8292, 'eval_samples_per_second': 27.152, 'eval_steps_per_second': 1.14, 'epoch': 0.36} +{'loss': 2.0105, 'grad_norm': 2.182343006134033, 'learning_rate': 4.912672623883022e-06, 'epoch': 0.36} +{'loss': 2.0024, 'grad_norm': 2.443370819091797, 'learning_rate': 4.910641754671e-06, 'epoch': 0.36} +{'loss': 2.0316, 'grad_norm': 2.509678363800049, 'learning_rate': 4.908610885458977e-06, 'epoch': 0.36} +{'loss': 2.0206, 'grad_norm': 2.4028046131134033, 'learning_rate': 4.906580016246954e-06, 'epoch': 0.37} +{'loss': 2.0749, 'grad_norm': 2.22725772857666, 'learning_rate': 4.9045491470349315e-06, 'epoch': 0.37} +{'loss': 2.0441, 'grad_norm': 2.4052486419677734, 'learning_rate': 4.902518277822909e-06, 'epoch': 0.37} +{'loss': 2.0522, 'grad_norm': 2.509493589401245, 'learning_rate': 4.900487408610886e-06, 'epoch': 0.37} +{'loss': 2.0579, 'grad_norm': 2.263730764389038, 'learning_rate': 4.898456539398863e-06, 'epoch': 0.37} +{'loss': 2.0712, 'grad_norm': 2.6925342082977295, 'learning_rate': 4.89642567018684e-06, 'epoch': 0.37} +{'loss': 2.1426, 'grad_norm': 2.517216920852661, 'learning_rate': 4.894394800974818e-06, 'epoch': 0.37} +{'eval_loss': 2.175614833831787, 'eval_runtime': 36.8047, 'eval_samples_per_second': 27.17, 'eval_steps_per_second': 1.141, 'epoch': 0.37} +{'loss': 1.9838, 'grad_norm': 2.222014904022217, 'learning_rate': 4.892363931762794e-06, 'epoch': 0.37} +{'loss': 2.0666, 'grad_norm': 2.2543673515319824, 'learning_rate': 4.8903330625507725e-06, 'epoch': 0.37} +{'loss': 2.0072, 'grad_norm': 2.496208906173706, 'learning_rate': 4.888302193338749e-06, 'epoch': 0.38} +{'loss': 1.9863, 'grad_norm': 2.194899559020996, 'learning_rate': 4.886271324126727e-06, 'epoch': 0.38} +{'loss': 2.0828, 'grad_norm': 2.666233539581299, 'learning_rate': 4.8842404549147034e-06, 'epoch': 0.38} +{'loss': 2.0716, 'grad_norm': 2.239429473876953, 'learning_rate': 4.8822095857026816e-06, 'epoch': 0.38} +{'loss': 2.0361, 'grad_norm': 2.4315106868743896, 'learning_rate': 4.880178716490658e-06, 'epoch': 0.38} +{'loss': 1.987, 'grad_norm': 2.616281509399414, 'learning_rate': 4.878147847278636e-06, 'epoch': 0.38} +{'loss': 1.9996, 'grad_norm': 2.1155405044555664, 'learning_rate': 4.8761169780666125e-06, 'epoch': 0.38} +{'loss': 1.9723, 'grad_norm': 2.784736156463623, 'learning_rate': 4.874086108854591e-06, 'epoch': 0.38} +{'eval_loss': 2.1697511672973633, 'eval_runtime': 36.8314, 'eval_samples_per_second': 27.151, 'eval_steps_per_second': 1.14, 'epoch': 0.38} +{'loss': 1.9823, 'grad_norm': 2.425434112548828, 'learning_rate': 4.872055239642567e-06, 'epoch': 0.38} +{'loss': 1.9657, 'grad_norm': 2.474745512008667, 'learning_rate': 4.870024370430544e-06, 'epoch': 0.39} +{'loss': 2.017, 'grad_norm': 2.5412509441375732, 'learning_rate': 4.867993501218522e-06, 'epoch': 0.39} +{'loss': 2.0564, 'grad_norm': 2.587143659591675, 'learning_rate': 4.865962632006499e-06, 'epoch': 0.39} +{'loss': 1.9895, 'grad_norm': 2.4585652351379395, 'learning_rate': 4.863931762794476e-06, 'epoch': 0.39} +{'loss': 2.0179, 'grad_norm': 3.121283769607544, 'learning_rate': 4.8619008935824535e-06, 'epoch': 0.39} +{'loss': 2.049, 'grad_norm': 2.432809352874756, 'learning_rate': 4.859870024370431e-06, 'epoch': 0.39} +{'loss': 2.0667, 'grad_norm': 2.6077938079833984, 'learning_rate': 4.857839155158408e-06, 'epoch': 0.39} +{'loss': 2.0776, 'grad_norm': 2.636944055557251, 'learning_rate': 4.855808285946385e-06, 'epoch': 0.39} +{'loss': 2.0533, 'grad_norm': 2.3640847206115723, 'learning_rate': 4.8537774167343626e-06, 'epoch': 0.39} +{'eval_loss': 2.165191888809204, 'eval_runtime': 36.8156, 'eval_samples_per_second': 27.162, 'eval_steps_per_second': 1.141, 'epoch': 0.39} +{'loss': 2.0374, 'grad_norm': 2.7718842029571533, 'learning_rate': 4.85174654752234e-06, 'epoch': 0.4} +{'loss': 2.035, 'grad_norm': 2.86139178276062, 'learning_rate': 4.849715678310317e-06, 'epoch': 0.4} +{'loss': 2.0307, 'grad_norm': 2.1344094276428223, 'learning_rate': 4.847684809098294e-06, 'epoch': 0.4} +{'loss': 2.0346, 'grad_norm': 2.383131742477417, 'learning_rate': 4.845653939886272e-06, 'epoch': 0.4} +{'loss': 2.0325, 'grad_norm': 2.606567144393921, 'learning_rate': 4.843623070674249e-06, 'epoch': 0.4} +{'loss': 2.0065, 'grad_norm': 2.1096670627593994, 'learning_rate': 4.841592201462226e-06, 'epoch': 0.4} +{'loss': 2.0643, 'grad_norm': 2.513556480407715, 'learning_rate': 4.8395613322502035e-06, 'epoch': 0.4} +{'loss': 1.9637, 'grad_norm': 2.236032485961914, 'learning_rate': 4.837530463038181e-06, 'epoch': 0.4} +{'loss': 2.0217, 'grad_norm': 2.1936111450195312, 'learning_rate': 4.835499593826158e-06, 'epoch': 0.4} +{'loss': 2.049, 'grad_norm': 2.11141037940979, 'learning_rate': 4.833468724614135e-06, 'epoch': 0.41} +{'eval_loss': 2.159877061843872, 'eval_runtime': 36.8393, 'eval_samples_per_second': 27.145, 'eval_steps_per_second': 1.14, 'epoch': 0.41} +{'loss': 2.0267, 'grad_norm': 2.2242090702056885, 'learning_rate': 4.831437855402113e-06, 'epoch': 0.41} +{'loss': 1.9266, 'grad_norm': 2.1799142360687256, 'learning_rate': 4.82940698619009e-06, 'epoch': 0.41} +{'loss': 1.9934, 'grad_norm': 2.0943312644958496, 'learning_rate': 4.827376116978067e-06, 'epoch': 0.41} +{'loss': 2.0524, 'grad_norm': 2.386090040206909, 'learning_rate': 4.8253452477660444e-06, 'epoch': 0.41} +{'loss': 2.0335, 'grad_norm': 2.259557008743286, 'learning_rate': 4.823314378554022e-06, 'epoch': 0.41} +{'loss': 2.0234, 'grad_norm': 2.342690944671631, 'learning_rate': 4.821283509341998e-06, 'epoch': 0.41} +{'loss': 1.9935, 'grad_norm': 2.207160472869873, 'learning_rate': 4.819252640129976e-06, 'epoch': 0.41} +{'loss': 2.0111, 'grad_norm': 2.3230350017547607, 'learning_rate': 4.817221770917953e-06, 'epoch': 0.41} +{'loss': 1.9941, 'grad_norm': 2.3750221729278564, 'learning_rate': 4.815190901705931e-06, 'epoch': 0.42} +{'loss': 2.0216, 'grad_norm': 2.139505624771118, 'learning_rate': 4.813160032493907e-06, 'epoch': 0.42} +{'eval_loss': 2.152115821838379, 'eval_runtime': 36.8435, 'eval_samples_per_second': 27.142, 'eval_steps_per_second': 1.14, 'epoch': 0.42} +{'loss': 2.0082, 'grad_norm': 2.3400697708129883, 'learning_rate': 4.811129163281885e-06, 'epoch': 0.42} +{'loss': 2.0431, 'grad_norm': 2.4719579219818115, 'learning_rate': 4.809098294069862e-06, 'epoch': 0.42} +{'loss': 2.0241, 'grad_norm': 2.1330277919769287, 'learning_rate': 4.80706742485784e-06, 'epoch': 0.42} +{'loss': 2.0795, 'grad_norm': 2.3635149002075195, 'learning_rate': 4.805036555645816e-06, 'epoch': 0.42} +{'loss': 2.001, 'grad_norm': 2.331961154937744, 'learning_rate': 4.8030056864337945e-06, 'epoch': 0.42} +{'loss': 2.0552, 'grad_norm': 2.593552589416504, 'learning_rate': 4.800974817221771e-06, 'epoch': 0.42} +{'loss': 2.0069, 'grad_norm': 2.3201818466186523, 'learning_rate': 4.798943948009749e-06, 'epoch': 0.42} +{'loss': 2.0232, 'grad_norm': 2.2886810302734375, 'learning_rate': 4.7969130787977254e-06, 'epoch': 0.43} +{'loss': 2.0149, 'grad_norm': 2.208707809448242, 'learning_rate': 4.794882209585703e-06, 'epoch': 0.43} +{'loss': 2.0436, 'grad_norm': 2.287991523742676, 'learning_rate': 4.79285134037368e-06, 'epoch': 0.43} +{'eval_loss': 2.1465628147125244, 'eval_runtime': 37.0388, 'eval_samples_per_second': 26.999, 'eval_steps_per_second': 1.134, 'epoch': 0.43} +{'loss': 1.9824, 'grad_norm': 2.0982491970062256, 'learning_rate': 4.790820471161657e-06, 'epoch': 0.43} +{'loss': 1.9914, 'grad_norm': 2.2861428260803223, 'learning_rate': 4.7887896019496345e-06, 'epoch': 0.43} +{'loss': 2.0057, 'grad_norm': 2.2473807334899902, 'learning_rate': 4.786758732737612e-06, 'epoch': 0.43} +{'loss': 2.0457, 'grad_norm': 2.1168124675750732, 'learning_rate': 4.784727863525589e-06, 'epoch': 0.43} +{'loss': 1.9618, 'grad_norm': 2.2538185119628906, 'learning_rate': 4.782696994313566e-06, 'epoch': 0.43} +{'loss': 2.0172, 'grad_norm': 2.1803009510040283, 'learning_rate': 4.780666125101544e-06, 'epoch': 0.43} +{'loss': 1.9548, 'grad_norm': 2.3871731758117676, 'learning_rate': 4.778635255889521e-06, 'epoch': 0.44} +{'loss': 1.978, 'grad_norm': 2.3013761043548584, 'learning_rate': 4.776604386677498e-06, 'epoch': 0.44} +{'loss': 1.9895, 'grad_norm': 2.259394645690918, 'learning_rate': 4.7745735174654755e-06, 'epoch': 0.44} +{'loss': 2.0485, 'grad_norm': 2.233656406402588, 'learning_rate': 4.772542648253453e-06, 'epoch': 0.44} +{'eval_loss': 2.146829128265381, 'eval_runtime': 37.1078, 'eval_samples_per_second': 26.948, 'eval_steps_per_second': 1.132, 'epoch': 0.44} +{'loss': 2.051, 'grad_norm': 2.131877899169922, 'learning_rate': 4.77051177904143e-06, 'epoch': 0.44} +{'loss': 1.977, 'grad_norm': 2.1987686157226562, 'learning_rate': 4.768480909829407e-06, 'epoch': 0.44} +{'loss': 2.0238, 'grad_norm': 2.26301646232605, 'learning_rate': 4.766450040617385e-06, 'epoch': 0.44} +{'loss': 1.9943, 'grad_norm': 2.288947105407715, 'learning_rate': 4.764419171405362e-06, 'epoch': 0.44} +{'loss': 2.0376, 'grad_norm': 2.364522695541382, 'learning_rate': 4.762388302193339e-06, 'epoch': 0.44} +{'loss': 2.0184, 'grad_norm': 2.1967415809631348, 'learning_rate': 4.760357432981316e-06, 'epoch': 0.44} +{'loss': 1.9862, 'grad_norm': 2.0930731296539307, 'learning_rate': 4.758326563769294e-06, 'epoch': 0.45} +{'loss': 2.022, 'grad_norm': 2.2198588848114014, 'learning_rate': 4.756295694557271e-06, 'epoch': 0.45} +{'loss': 2.0166, 'grad_norm': 2.0918731689453125, 'learning_rate': 4.754264825345248e-06, 'epoch': 0.45} +{'loss': 1.9502, 'grad_norm': 2.40588116645813, 'learning_rate': 4.7522339561332255e-06, 'epoch': 0.45} +{'eval_loss': 2.1474976539611816, 'eval_runtime': 36.8377, 'eval_samples_per_second': 27.146, 'eval_steps_per_second': 1.14, 'epoch': 0.45} +{'loss': 1.9715, 'grad_norm': 2.228062629699707, 'learning_rate': 4.750203086921203e-06, 'epoch': 0.45} +{'loss': 1.9359, 'grad_norm': 2.216545820236206, 'learning_rate': 4.74817221770918e-06, 'epoch': 0.45} +{'loss': 2.0099, 'grad_norm': 2.3648414611816406, 'learning_rate': 4.7461413484971565e-06, 'epoch': 0.45} +{'loss': 2.0623, 'grad_norm': 2.2856826782226562, 'learning_rate': 4.744110479285135e-06, 'epoch': 0.45} +{'loss': 1.9527, 'grad_norm': 2.5282676219940186, 'learning_rate': 4.742079610073111e-06, 'epoch': 0.45} +{'loss': 1.9769, 'grad_norm': 2.1065621376037598, 'learning_rate': 4.740048740861089e-06, 'epoch': 0.46} +{'loss': 1.9881, 'grad_norm': 2.3433425426483154, 'learning_rate': 4.738017871649066e-06, 'epoch': 0.46} +{'loss': 1.9982, 'grad_norm': 2.2476186752319336, 'learning_rate': 4.735987002437044e-06, 'epoch': 0.46} +{'loss': 2.0616, 'grad_norm': 2.3849618434906006, 'learning_rate': 4.73395613322502e-06, 'epoch': 0.46} +{'loss': 2.0686, 'grad_norm': 2.398130416870117, 'learning_rate': 4.731925264012998e-06, 'epoch': 0.46} +{'eval_loss': 2.1412012577056885, 'eval_runtime': 36.8484, 'eval_samples_per_second': 27.138, 'eval_steps_per_second': 1.14, 'epoch': 0.46} +{'loss': 2.0278, 'grad_norm': 2.224095106124878, 'learning_rate': 4.729894394800975e-06, 'epoch': 0.46} +{'loss': 1.9633, 'grad_norm': 2.177316665649414, 'learning_rate': 4.727863525588953e-06, 'epoch': 0.46} +{'loss': 1.9211, 'grad_norm': 2.212502956390381, 'learning_rate': 4.725832656376929e-06, 'epoch': 0.46} +{'loss': 1.9822, 'grad_norm': 2.329396963119507, 'learning_rate': 4.723801787164907e-06, 'epoch': 0.46} +{'loss': 2.0233, 'grad_norm': 2.351280689239502, 'learning_rate': 4.721770917952885e-06, 'epoch': 0.47} +{'loss': 1.9609, 'grad_norm': 2.1245787143707275, 'learning_rate': 4.719740048740861e-06, 'epoch': 0.47} +{'loss': 1.9905, 'grad_norm': 2.1225812435150146, 'learning_rate': 4.717709179528839e-06, 'epoch': 0.47} +{'loss': 2.0114, 'grad_norm': 2.0605881214141846, 'learning_rate': 4.715678310316816e-06, 'epoch': 0.47} +{'loss': 2.0146, 'grad_norm': 2.241417169570923, 'learning_rate': 4.713647441104794e-06, 'epoch': 0.47} +{'loss': 1.9548, 'grad_norm': 2.2796316146850586, 'learning_rate': 4.71161657189277e-06, 'epoch': 0.47} +{'eval_loss': 2.134554862976074, 'eval_runtime': 36.8374, 'eval_samples_per_second': 27.146, 'eval_steps_per_second': 1.14, 'epoch': 0.47} +{'loss': 1.9465, 'grad_norm': 2.4043936729431152, 'learning_rate': 4.709585702680748e-06, 'epoch': 0.47} +{'loss': 2.0234, 'grad_norm': 2.176814317703247, 'learning_rate': 4.707554833468725e-06, 'epoch': 0.47} +{'loss': 2.0795, 'grad_norm': 2.3174710273742676, 'learning_rate': 4.705523964256703e-06, 'epoch': 0.47} +{'loss': 1.9687, 'grad_norm': 2.460334539413452, 'learning_rate': 4.703493095044679e-06, 'epoch': 0.48} +{'loss': 1.9911, 'grad_norm': 2.4824578762054443, 'learning_rate': 4.701462225832657e-06, 'epoch': 0.48} +{'loss': 1.9128, 'grad_norm': 2.5180532932281494, 'learning_rate': 4.699431356620634e-06, 'epoch': 0.48} +{'loss': 2.071, 'grad_norm': 2.380042791366577, 'learning_rate': 4.697400487408612e-06, 'epoch': 0.48} +{'loss': 2.0143, 'grad_norm': 2.441718816757202, 'learning_rate': 4.695369618196588e-06, 'epoch': 0.48} +{'loss': 1.9867, 'grad_norm': 2.9212632179260254, 'learning_rate': 4.693338748984566e-06, 'epoch': 0.48} +{'loss': 1.9852, 'grad_norm': 2.3351190090179443, 'learning_rate': 4.691307879772543e-06, 'epoch': 0.48} +{'eval_loss': 2.1334691047668457, 'eval_runtime': 36.8617, 'eval_samples_per_second': 27.128, 'eval_steps_per_second': 1.139, 'epoch': 0.48} +{'loss': 1.9353, 'grad_norm': 3.4289700984954834, 'learning_rate': 4.68927701056052e-06, 'epoch': 0.48} +{'loss': 2.0259, 'grad_norm': 2.154865026473999, 'learning_rate': 4.6872461413484975e-06, 'epoch': 0.48} +{'loss': 2.0101, 'grad_norm': 2.4053304195404053, 'learning_rate': 4.685215272136475e-06, 'epoch': 0.49} +{'loss': 1.9976, 'grad_norm': 2.250504493713379, 'learning_rate': 4.683184402924452e-06, 'epoch': 0.49} +{'loss': 1.9925, 'grad_norm': 2.9280881881713867, 'learning_rate': 4.681153533712429e-06, 'epoch': 0.49} +{'loss': 2.0534, 'grad_norm': 2.3568203449249268, 'learning_rate': 4.679122664500407e-06, 'epoch': 0.49} +{'loss': 1.9486, 'grad_norm': 3.101870536804199, 'learning_rate': 4.677091795288384e-06, 'epoch': 0.49} +{'loss': 1.9334, 'grad_norm': 2.2512426376342773, 'learning_rate': 4.675060926076361e-06, 'epoch': 0.49} +{'loss': 2.0329, 'grad_norm': 2.2165493965148926, 'learning_rate': 4.673030056864338e-06, 'epoch': 0.49} +{'loss': 1.9409, 'grad_norm': 2.2993414402008057, 'learning_rate': 4.670999187652316e-06, 'epoch': 0.49} +{'eval_loss': 2.1301963329315186, 'eval_runtime': 36.8396, 'eval_samples_per_second': 27.145, 'eval_steps_per_second': 1.14, 'epoch': 0.49} +{'loss': 1.9416, 'grad_norm': 2.196807622909546, 'learning_rate': 4.668968318440293e-06, 'epoch': 0.49} +{'loss': 1.9426, 'grad_norm': 2.04422926902771, 'learning_rate': 4.66693744922827e-06, 'epoch': 0.5} +{'loss': 2.0223, 'grad_norm': 2.2703120708465576, 'learning_rate': 4.6649065800162475e-06, 'epoch': 0.5} +{'loss': 1.9215, 'grad_norm': 2.350550889968872, 'learning_rate': 4.662875710804225e-06, 'epoch': 0.5} +{'loss': 1.9945, 'grad_norm': 2.192880868911743, 'learning_rate': 4.660844841592202e-06, 'epoch': 0.5} +{'loss': 1.9014, 'grad_norm': 2.3187272548675537, 'learning_rate': 4.658813972380179e-06, 'epoch': 0.5} +{'loss': 2.0314, 'grad_norm': 2.3140294551849365, 'learning_rate': 4.656783103168157e-06, 'epoch': 0.5} +{'loss': 1.896, 'grad_norm': 2.1471943855285645, 'learning_rate': 4.654752233956134e-06, 'epoch': 0.5} +{'loss': 1.9878, 'grad_norm': 2.2951014041900635, 'learning_rate': 4.652721364744111e-06, 'epoch': 0.5} +{'loss': 1.9573, 'grad_norm': 2.2135801315307617, 'learning_rate': 4.6506904955320885e-06, 'epoch': 0.5} +{'eval_loss': 2.135219097137451, 'eval_runtime': 36.8713, 'eval_samples_per_second': 27.121, 'eval_steps_per_second': 1.139, 'epoch': 0.5} +{'loss': 2.0127, 'grad_norm': 2.1129870414733887, 'learning_rate': 4.648659626320066e-06, 'epoch': 0.51} +{'loss': 2.0106, 'grad_norm': 2.299039602279663, 'learning_rate': 4.646628757108043e-06, 'epoch': 0.51} +{'loss': 1.9812, 'grad_norm': 2.299088716506958, 'learning_rate': 4.6445978878960194e-06, 'epoch': 0.51} +{'loss': 1.9435, 'grad_norm': 2.0711328983306885, 'learning_rate': 4.6425670186839976e-06, 'epoch': 0.51} +{'loss': 1.9793, 'grad_norm': 2.1749722957611084, 'learning_rate': 4.640536149471974e-06, 'epoch': 0.51} +{'loss': 2.0117, 'grad_norm': 2.127612352371216, 'learning_rate': 4.638505280259952e-06, 'epoch': 0.51} +{'loss': 2.0336, 'grad_norm': 2.1723556518554688, 'learning_rate': 4.6364744110479285e-06, 'epoch': 0.51} +{'loss': 1.9861, 'grad_norm': 2.1473746299743652, 'learning_rate': 4.634443541835907e-06, 'epoch': 0.51} +{'loss': 2.0428, 'grad_norm': 2.2084238529205322, 'learning_rate': 4.632412672623883e-06, 'epoch': 0.51} +{'loss': 1.9594, 'grad_norm': 2.3061389923095703, 'learning_rate': 4.630381803411861e-06, 'epoch': 0.52} +{'eval_loss': 2.12579607963562, 'eval_runtime': 36.8987, 'eval_samples_per_second': 27.101, 'eval_steps_per_second': 1.138, 'epoch': 0.52} +{'loss': 1.8626, 'grad_norm': 2.340785264968872, 'learning_rate': 4.628350934199838e-06, 'epoch': 0.52} +{'loss': 1.9679, 'grad_norm': 2.1136975288391113, 'learning_rate': 4.626320064987816e-06, 'epoch': 0.52} +{'loss': 2.001, 'grad_norm': 2.311692714691162, 'learning_rate': 4.624289195775792e-06, 'epoch': 0.52} +{'loss': 1.9731, 'grad_norm': 2.2172560691833496, 'learning_rate': 4.62225832656377e-06, 'epoch': 0.52} +{'loss': 1.9804, 'grad_norm': 2.2663748264312744, 'learning_rate': 4.620227457351747e-06, 'epoch': 0.52} +{'loss': 1.972, 'grad_norm': 2.2165415287017822, 'learning_rate': 4.618196588139724e-06, 'epoch': 0.52} +{'loss': 1.9886, 'grad_norm': 2.1509106159210205, 'learning_rate': 4.616165718927701e-06, 'epoch': 0.52} +{'loss': 1.9412, 'grad_norm': 2.096590280532837, 'learning_rate': 4.6141348497156786e-06, 'epoch': 0.52} +{'loss': 2.0331, 'grad_norm': 2.123234987258911, 'learning_rate': 4.612103980503656e-06, 'epoch': 0.52} +{'loss': 2.0127, 'grad_norm': 2.3646109104156494, 'learning_rate': 4.610073111291633e-06, 'epoch': 0.53} +{'eval_loss': 2.1210875511169434, 'eval_runtime': 36.8649, 'eval_samples_per_second': 27.126, 'eval_steps_per_second': 1.139, 'epoch': 0.53} +{'loss': 1.9867, 'grad_norm': 2.15993070602417, 'learning_rate': 4.60804224207961e-06, 'epoch': 0.53} +{'loss': 1.9611, 'grad_norm': 2.0517752170562744, 'learning_rate': 4.606011372867588e-06, 'epoch': 0.53} +{'loss': 1.9939, 'grad_norm': 2.262526035308838, 'learning_rate': 4.603980503655565e-06, 'epoch': 0.53} +{'loss': 1.9674, 'grad_norm': 2.203848361968994, 'learning_rate': 4.601949634443542e-06, 'epoch': 0.53} +{'loss': 1.902, 'grad_norm': 2.0661096572875977, 'learning_rate': 4.5999187652315195e-06, 'epoch': 0.53} +{'loss': 1.9026, 'grad_norm': 2.0441699028015137, 'learning_rate': 4.597887896019497e-06, 'epoch': 0.53} +{'loss': 1.9594, 'grad_norm': 2.330298900604248, 'learning_rate': 4.595857026807474e-06, 'epoch': 0.53} +{'loss': 1.969, 'grad_norm': 2.0567939281463623, 'learning_rate': 4.593826157595451e-06, 'epoch': 0.53} +{'loss': 1.9501, 'grad_norm': 2.2553024291992188, 'learning_rate': 4.591795288383429e-06, 'epoch': 0.54} +{'loss': 2.0365, 'grad_norm': 2.525721549987793, 'learning_rate': 4.589764419171406e-06, 'epoch': 0.54} +{'eval_loss': 2.116159200668335, 'eval_runtime': 37.0471, 'eval_samples_per_second': 26.993, 'eval_steps_per_second': 1.134, 'epoch': 0.54} +{'loss': 1.9768, 'grad_norm': 2.017068386077881, 'learning_rate': 4.587733549959383e-06, 'epoch': 0.54} +{'loss': 1.9499, 'grad_norm': 2.3081953525543213, 'learning_rate': 4.5857026807473604e-06, 'epoch': 0.54} +{'loss': 1.9289, 'grad_norm': 2.317981243133545, 'learning_rate': 4.583671811535338e-06, 'epoch': 0.54} +{'loss': 1.9095, 'grad_norm': 2.2007813453674316, 'learning_rate': 4.581640942323315e-06, 'epoch': 0.54} +{'loss': 2.0063, 'grad_norm': 2.251990795135498, 'learning_rate': 4.579610073111292e-06, 'epoch': 0.54} +{'loss': 2.0236, 'grad_norm': 2.774461507797241, 'learning_rate': 4.5775792038992695e-06, 'epoch': 0.54} +{'loss': 2.0146, 'grad_norm': 2.346486806869507, 'learning_rate': 4.575548334687247e-06, 'epoch': 0.54} +{'loss': 2.0431, 'grad_norm': 2.2475638389587402, 'learning_rate': 4.573517465475224e-06, 'epoch': 0.55} +{'loss': 1.943, 'grad_norm': 2.385378122329712, 'learning_rate': 4.571486596263201e-06, 'epoch': 0.55} +{'loss': 1.9989, 'grad_norm': 2.4229276180267334, 'learning_rate': 4.569455727051178e-06, 'epoch': 0.55} +{'eval_loss': 2.116908311843872, 'eval_runtime': 37.1237, 'eval_samples_per_second': 26.937, 'eval_steps_per_second': 1.131, 'epoch': 0.55} +{'loss': 1.9964, 'grad_norm': 2.272860527038574, 'learning_rate': 4.567424857839156e-06, 'epoch': 0.55} +{'loss': 1.9761, 'grad_norm': 2.585275888442993, 'learning_rate': 4.565393988627132e-06, 'epoch': 0.55} +{'loss': 2.0429, 'grad_norm': 2.2231333255767822, 'learning_rate': 4.5633631194151105e-06, 'epoch': 0.55} +{'loss': 1.9702, 'grad_norm': 2.550520181655884, 'learning_rate': 4.561332250203087e-06, 'epoch': 0.55} +{'loss': 1.9705, 'grad_norm': 2.5849199295043945, 'learning_rate': 4.559301380991065e-06, 'epoch': 0.55} +{'loss': 1.9109, 'grad_norm': 2.135770559310913, 'learning_rate': 4.5572705117790414e-06, 'epoch': 0.55} +{'loss': 1.9574, 'grad_norm': 2.49159574508667, 'learning_rate': 4.5552396425670196e-06, 'epoch': 0.56} +{'loss': 2.0113, 'grad_norm': 2.4807381629943848, 'learning_rate': 4.553208773354996e-06, 'epoch': 0.56} +{'loss': 1.9272, 'grad_norm': 2.373417854309082, 'learning_rate': 4.551177904142974e-06, 'epoch': 0.56} +{'loss': 2.0213, 'grad_norm': 2.057331085205078, 'learning_rate': 4.5491470349309505e-06, 'epoch': 0.56} +{'eval_loss': 2.115997552871704, 'eval_runtime': 36.8705, 'eval_samples_per_second': 27.122, 'eval_steps_per_second': 1.139, 'epoch': 0.56} +{'loss': 1.9467, 'grad_norm': 2.352627754211426, 'learning_rate': 4.547116165718929e-06, 'epoch': 0.56} +{'loss': 1.9719, 'grad_norm': 2.4589340686798096, 'learning_rate': 4.545085296506905e-06, 'epoch': 0.56} +{'loss': 1.9388, 'grad_norm': 2.0117125511169434, 'learning_rate': 4.543054427294882e-06, 'epoch': 0.56} +{'loss': 1.9214, 'grad_norm': 2.0877344608306885, 'learning_rate': 4.54102355808286e-06, 'epoch': 0.56} +{'loss': 1.9312, 'grad_norm': 2.4225330352783203, 'learning_rate': 4.538992688870837e-06, 'epoch': 0.56} +{'loss': 1.9588, 'grad_norm': 2.2232658863067627, 'learning_rate': 4.536961819658814e-06, 'epoch': 0.57} +{'loss': 1.9528, 'grad_norm': 2.0866599082946777, 'learning_rate': 4.5349309504467915e-06, 'epoch': 0.57} +{'loss': 1.9834, 'grad_norm': 2.272505760192871, 'learning_rate': 4.532900081234769e-06, 'epoch': 0.57} +{'loss': 1.9479, 'grad_norm': 2.2399017810821533, 'learning_rate': 4.530869212022746e-06, 'epoch': 0.57} +{'loss': 1.9756, 'grad_norm': 2.8384666442871094, 'learning_rate': 4.528838342810723e-06, 'epoch': 0.57} +{'eval_loss': 2.111968517303467, 'eval_runtime': 36.8132, 'eval_samples_per_second': 27.164, 'eval_steps_per_second': 1.141, 'epoch': 0.57} +{'loss': 1.9594, 'grad_norm': 2.231113910675049, 'learning_rate': 4.526807473598701e-06, 'epoch': 0.57} +{'loss': 1.9669, 'grad_norm': 2.4858415126800537, 'learning_rate': 4.524776604386678e-06, 'epoch': 0.57} +{'loss': 1.9429, 'grad_norm': 2.2841124534606934, 'learning_rate': 4.522745735174655e-06, 'epoch': 0.57} +{'loss': 1.9953, 'grad_norm': 2.0490520000457764, 'learning_rate': 4.520714865962632e-06, 'epoch': 0.57} +{'loss': 1.9729, 'grad_norm': 2.2872908115386963, 'learning_rate': 4.51868399675061e-06, 'epoch': 0.58} +{'loss': 1.9891, 'grad_norm': 2.2335281372070312, 'learning_rate': 4.516653127538587e-06, 'epoch': 0.58} +{'loss': 1.9387, 'grad_norm': 2.0251572132110596, 'learning_rate': 4.514622258326564e-06, 'epoch': 0.58} +{'loss': 1.9275, 'grad_norm': 2.3072011470794678, 'learning_rate': 4.5125913891145415e-06, 'epoch': 0.58} +{'loss': 1.9524, 'grad_norm': 2.4449033737182617, 'learning_rate': 4.510560519902519e-06, 'epoch': 0.58} +{'loss': 1.99, 'grad_norm': 2.2075350284576416, 'learning_rate': 4.508529650690496e-06, 'epoch': 0.58} +{'eval_loss': 2.1130809783935547, 'eval_runtime': 36.8325, 'eval_samples_per_second': 27.15, 'eval_steps_per_second': 1.14, 'epoch': 0.58} +{'loss': 1.9133, 'grad_norm': 2.148533821105957, 'learning_rate': 4.506498781478473e-06, 'epoch': 0.58} +{'loss': 1.9131, 'grad_norm': 2.2660858631134033, 'learning_rate': 4.504467912266451e-06, 'epoch': 0.58} +{'loss': 1.955, 'grad_norm': 2.31901478767395, 'learning_rate': 4.502437043054428e-06, 'epoch': 0.58} +{'loss': 2.0227, 'grad_norm': 2.0394129753112793, 'learning_rate': 4.500406173842405e-06, 'epoch': 0.59} +{'loss': 1.9819, 'grad_norm': 2.2833235263824463, 'learning_rate': 4.4983753046303824e-06, 'epoch': 0.59} +{'loss': 1.9587, 'grad_norm': 2.2009031772613525, 'learning_rate': 4.49634443541836e-06, 'epoch': 0.59} +{'loss': 1.9458, 'grad_norm': 2.1384377479553223, 'learning_rate': 4.494313566206336e-06, 'epoch': 0.59} +{'loss': 2.0082, 'grad_norm': 2.2180237770080566, 'learning_rate': 4.492282696994314e-06, 'epoch': 0.59} +{'loss': 1.9423, 'grad_norm': 2.1079695224761963, 'learning_rate': 4.490251827782291e-06, 'epoch': 0.59} +{'loss': 1.911, 'grad_norm': 2.3531908988952637, 'learning_rate': 4.488220958570269e-06, 'epoch': 0.59} +{'eval_loss': 2.1106293201446533, 'eval_runtime': 36.8449, 'eval_samples_per_second': 27.141, 'eval_steps_per_second': 1.14, 'epoch': 0.59} +{'loss': 1.942, 'grad_norm': 2.173271417617798, 'learning_rate': 4.486190089358245e-06, 'epoch': 0.59} +{'loss': 1.8837, 'grad_norm': 2.161085367202759, 'learning_rate': 4.484159220146223e-06, 'epoch': 0.59} +{'loss': 1.9926, 'grad_norm': 2.1999270915985107, 'learning_rate': 4.4821283509342e-06, 'epoch': 0.6} +{'loss': 1.9738, 'grad_norm': 2.0908172130584717, 'learning_rate': 4.480097481722178e-06, 'epoch': 0.6} +{'loss': 1.9485, 'grad_norm': 2.1115596294403076, 'learning_rate': 4.478066612510154e-06, 'epoch': 0.6} +{'loss': 1.9171, 'grad_norm': 2.0752980709075928, 'learning_rate': 4.4760357432981325e-06, 'epoch': 0.6} +{'loss': 1.9363, 'grad_norm': 2.1090850830078125, 'learning_rate': 4.474004874086109e-06, 'epoch': 0.6} +{'loss': 2.079, 'grad_norm': 2.2388253211975098, 'learning_rate': 4.471974004874087e-06, 'epoch': 0.6} +{'loss': 1.9066, 'grad_norm': 2.13826584815979, 'learning_rate': 4.4699431356620635e-06, 'epoch': 0.6} +{'loss': 2.0115, 'grad_norm': 2.139592409133911, 'learning_rate': 4.467912266450041e-06, 'epoch': 0.6} +{'eval_loss': 2.107339382171631, 'eval_runtime': 36.8549, 'eval_samples_per_second': 27.133, 'eval_steps_per_second': 1.14, 'epoch': 0.6} +{'loss': 1.953, 'grad_norm': 2.2159054279327393, 'learning_rate': 4.465881397238018e-06, 'epoch': 0.6} +{'loss': 1.9418, 'grad_norm': 2.0024075508117676, 'learning_rate': 4.463850528025995e-06, 'epoch': 0.6} +{'loss': 1.959, 'grad_norm': 2.0435662269592285, 'learning_rate': 4.4618196588139726e-06, 'epoch': 0.61} +{'loss': 1.9876, 'grad_norm': 2.204221487045288, 'learning_rate': 4.45978878960195e-06, 'epoch': 0.61} +{'loss': 1.976, 'grad_norm': 2.1536989212036133, 'learning_rate': 4.457757920389927e-06, 'epoch': 0.61} +{'loss': 1.9185, 'grad_norm': 2.1285719871520996, 'learning_rate': 4.455727051177904e-06, 'epoch': 0.61} +{'loss': 1.9482, 'grad_norm': 2.1217541694641113, 'learning_rate': 4.453696181965882e-06, 'epoch': 0.61} +{'loss': 2.0028, 'grad_norm': 2.1897120475769043, 'learning_rate': 4.451665312753859e-06, 'epoch': 0.61} +{'loss': 1.9683, 'grad_norm': 2.0617103576660156, 'learning_rate': 4.449634443541836e-06, 'epoch': 0.61} +{'loss': 2.0174, 'grad_norm': 2.091261863708496, 'learning_rate': 4.4476035743298135e-06, 'epoch': 0.61} +{'eval_loss': 2.1050779819488525, 'eval_runtime': 36.8771, 'eval_samples_per_second': 27.117, 'eval_steps_per_second': 1.139, 'epoch': 0.61} +{'loss': 1.9247, 'grad_norm': 2.2024059295654297, 'learning_rate': 4.445572705117791e-06, 'epoch': 0.61} +{'loss': 1.9419, 'grad_norm': 2.234591484069824, 'learning_rate': 4.443541835905768e-06, 'epoch': 0.62} +{'loss': 2.0076, 'grad_norm': 2.073786497116089, 'learning_rate': 4.441510966693745e-06, 'epoch': 0.62} +{'loss': 1.9475, 'grad_norm': 2.250511407852173, 'learning_rate': 4.439480097481723e-06, 'epoch': 0.62} +{'loss': 2.0042, 'grad_norm': 2.1097965240478516, 'learning_rate': 4.4374492282697e-06, 'epoch': 0.62} +{'loss': 1.8782, 'grad_norm': 2.1740925312042236, 'learning_rate': 4.435418359057677e-06, 'epoch': 0.62} +{'loss': 1.9604, 'grad_norm': 2.072244644165039, 'learning_rate': 4.433387489845654e-06, 'epoch': 0.62} +{'loss': 1.9673, 'grad_norm': 2.042480707168579, 'learning_rate': 4.431356620633632e-06, 'epoch': 0.62} +{'loss': 1.9208, 'grad_norm': 2.0622525215148926, 'learning_rate': 4.429325751421609e-06, 'epoch': 0.62} +{'loss': 1.9273, 'grad_norm': 2.120314836502075, 'learning_rate': 4.427294882209586e-06, 'epoch': 0.62} +{'eval_loss': 2.10455322265625, 'eval_runtime': 36.8351, 'eval_samples_per_second': 27.148, 'eval_steps_per_second': 1.14, 'epoch': 0.62} +{'loss': 1.9538, 'grad_norm': 2.1246325969696045, 'learning_rate': 4.4252640129975635e-06, 'epoch': 0.63} +{'loss': 1.8853, 'grad_norm': 2.1085169315338135, 'learning_rate': 4.423233143785541e-06, 'epoch': 0.63} +{'loss': 1.9515, 'grad_norm': 2.0212948322296143, 'learning_rate': 4.421202274573518e-06, 'epoch': 0.63} +{'loss': 1.9543, 'grad_norm': 2.0736632347106934, 'learning_rate': 4.4191714053614945e-06, 'epoch': 0.63} +{'loss': 1.854, 'grad_norm': 2.0515239238739014, 'learning_rate': 4.417140536149473e-06, 'epoch': 0.63} +{'loss': 1.8974, 'grad_norm': 2.026869058609009, 'learning_rate': 4.415109666937449e-06, 'epoch': 0.63} +{'loss': 1.9686, 'grad_norm': 2.0393314361572266, 'learning_rate': 4.413078797725427e-06, 'epoch': 0.63} +{'loss': 1.9154, 'grad_norm': 2.063215494155884, 'learning_rate': 4.411047928513404e-06, 'epoch': 0.63} +{'loss': 1.9585, 'grad_norm': 2.167123556137085, 'learning_rate': 4.409017059301382e-06, 'epoch': 0.63} +{'loss': 1.9644, 'grad_norm': 2.155423879623413, 'learning_rate': 4.406986190089358e-06, 'epoch': 0.64} +{'eval_loss': 2.101809024810791, 'eval_runtime': 36.8171, 'eval_samples_per_second': 27.161, 'eval_steps_per_second': 1.141, 'epoch': 0.64} +{'loss': 1.9824, 'grad_norm': 2.0016093254089355, 'learning_rate': 4.404955320877336e-06, 'epoch': 0.64} +{'loss': 1.9713, 'grad_norm': 2.0963051319122314, 'learning_rate': 4.402924451665313e-06, 'epoch': 0.64} +{'loss': 1.9263, 'grad_norm': 1.9784541130065918, 'learning_rate': 4.400893582453291e-06, 'epoch': 0.64} +{'loss': 1.9, 'grad_norm': 1.9902760982513428, 'learning_rate': 4.398862713241267e-06, 'epoch': 0.64} +{'loss': 1.9422, 'grad_norm': 2.1675639152526855, 'learning_rate': 4.396831844029245e-06, 'epoch': 0.64} +{'loss': 1.9559, 'grad_norm': 2.028651237487793, 'learning_rate': 4.394800974817222e-06, 'epoch': 0.64} +{'loss': 1.945, 'grad_norm': 2.1689412593841553, 'learning_rate': 4.392770105605199e-06, 'epoch': 0.64} +{'loss': 1.9596, 'grad_norm': 2.1114044189453125, 'learning_rate': 4.390739236393176e-06, 'epoch': 0.64} +{'loss': 1.9712, 'grad_norm': 2.117853879928589, 'learning_rate': 4.388708367181154e-06, 'epoch': 0.65} +{'loss': 1.9746, 'grad_norm': 1.9970754384994507, 'learning_rate': 4.386677497969131e-06, 'epoch': 0.65} +{'eval_loss': 2.0976545810699463, 'eval_runtime': 37.034, 'eval_samples_per_second': 27.002, 'eval_steps_per_second': 1.134, 'epoch': 0.65} +{'loss': 1.9602, 'grad_norm': 2.060075044631958, 'learning_rate': 4.384646628757108e-06, 'epoch': 0.65} +{'loss': 2.0127, 'grad_norm': 1.9594982862472534, 'learning_rate': 4.3826157595450855e-06, 'epoch': 0.65} +{'loss': 1.9324, 'grad_norm': 2.054703950881958, 'learning_rate': 4.380584890333063e-06, 'epoch': 0.65} +{'loss': 1.8969, 'grad_norm': 2.041199207305908, 'learning_rate': 4.37855402112104e-06, 'epoch': 0.65} +{'loss': 1.8449, 'grad_norm': 2.0088963508605957, 'learning_rate': 4.376523151909017e-06, 'epoch': 0.65} +{'loss': 2.0027, 'grad_norm': 2.000422477722168, 'learning_rate': 4.3744922826969946e-06, 'epoch': 0.65} +{'loss': 1.9225, 'grad_norm': 2.008295774459839, 'learning_rate': 4.372461413484972e-06, 'epoch': 0.65} +{'loss': 1.9575, 'grad_norm': 2.252830982208252, 'learning_rate': 4.370430544272949e-06, 'epoch': 0.66} +{'loss': 1.9791, 'grad_norm': 2.0865542888641357, 'learning_rate': 4.368399675060926e-06, 'epoch': 0.66} +{'loss': 1.993, 'grad_norm': 2.2093706130981445, 'learning_rate': 4.366368805848904e-06, 'epoch': 0.66} +{'eval_loss': 2.0949337482452393, 'eval_runtime': 37.114, 'eval_samples_per_second': 26.944, 'eval_steps_per_second': 1.132, 'epoch': 0.66} +{'loss': 1.9364, 'grad_norm': 2.096261739730835, 'learning_rate': 4.364337936636881e-06, 'epoch': 0.66} +{'loss': 1.8922, 'grad_norm': 2.3028810024261475, 'learning_rate': 4.362307067424858e-06, 'epoch': 0.66} +{'loss': 1.9572, 'grad_norm': 2.106886863708496, 'learning_rate': 4.3602761982128355e-06, 'epoch': 0.66} +{'loss': 1.9132, 'grad_norm': 2.1422243118286133, 'learning_rate': 4.358245329000813e-06, 'epoch': 0.66} +{'loss': 1.9356, 'grad_norm': 2.0614800453186035, 'learning_rate': 4.35621445978879e-06, 'epoch': 0.66} +{'loss': 1.9516, 'grad_norm': 2.013725996017456, 'learning_rate': 4.354183590576767e-06, 'epoch': 0.66} +{'loss': 1.9341, 'grad_norm': 2.1779627799987793, 'learning_rate': 4.352152721364745e-06, 'epoch': 0.67} +{'loss': 1.8796, 'grad_norm': 2.1698031425476074, 'learning_rate': 4.350121852152722e-06, 'epoch': 0.67} +{'loss': 1.9606, 'grad_norm': 2.1617958545684814, 'learning_rate': 4.348090982940699e-06, 'epoch': 0.67} +{'loss': 1.9491, 'grad_norm': 2.189724922180176, 'learning_rate': 4.3460601137286764e-06, 'epoch': 0.67} +{'eval_loss': 2.0930235385894775, 'eval_runtime': 36.816, 'eval_samples_per_second': 27.162, 'eval_steps_per_second': 1.141, 'epoch': 0.67} +{'loss': 1.971, 'grad_norm': 2.300693988800049, 'learning_rate': 4.344029244516654e-06, 'epoch': 0.67} +{'loss': 1.9289, 'grad_norm': 2.2398195266723633, 'learning_rate': 4.341998375304631e-06, 'epoch': 0.67} +{'loss': 2.0109, 'grad_norm': 2.0529696941375732, 'learning_rate': 4.339967506092607e-06, 'epoch': 0.67} +{'loss': 1.9443, 'grad_norm': 2.0417604446411133, 'learning_rate': 4.3379366368805855e-06, 'epoch': 0.67} +{'loss': 1.942, 'grad_norm': 2.1295909881591797, 'learning_rate': 4.335905767668562e-06, 'epoch': 0.67} +{'loss': 1.9272, 'grad_norm': 2.1724905967712402, 'learning_rate': 4.33387489845654e-06, 'epoch': 0.68} +{'loss': 1.9628, 'grad_norm': 2.078840494155884, 'learning_rate': 4.3318440292445165e-06, 'epoch': 0.68} +{'loss': 1.9398, 'grad_norm': 2.073406934738159, 'learning_rate': 4.329813160032495e-06, 'epoch': 0.68} +{'loss': 1.911, 'grad_norm': 2.3771591186523438, 'learning_rate': 4.327782290820471e-06, 'epoch': 0.68} +{'loss': 1.9037, 'grad_norm': 2.0965006351470947, 'learning_rate': 4.325751421608449e-06, 'epoch': 0.68} +{'eval_loss': 2.096085548400879, 'eval_runtime': 36.8754, 'eval_samples_per_second': 27.118, 'eval_steps_per_second': 1.139, 'epoch': 0.68} +{'loss': 1.9794, 'grad_norm': 2.189476490020752, 'learning_rate': 4.323720552396426e-06, 'epoch': 0.68} +{'loss': 1.9224, 'grad_norm': 2.453953504562378, 'learning_rate': 4.321689683184404e-06, 'epoch': 0.68} +{'loss': 1.9195, 'grad_norm': 2.0869131088256836, 'learning_rate': 4.31965881397238e-06, 'epoch': 0.68} +{'loss': 1.9214, 'grad_norm': 2.181950807571411, 'learning_rate': 4.3176279447603574e-06, 'epoch': 0.68} +{'loss': 1.9694, 'grad_norm': 2.0392794609069824, 'learning_rate': 4.315597075548335e-06, 'epoch': 0.68} +{'loss': 1.9623, 'grad_norm': 2.268585205078125, 'learning_rate': 4.313566206336312e-06, 'epoch': 0.69} +{'loss': 1.9815, 'grad_norm': 2.0459189414978027, 'learning_rate': 4.311535337124289e-06, 'epoch': 0.69} +{'loss': 1.9296, 'grad_norm': 2.1525423526763916, 'learning_rate': 4.3095044679122665e-06, 'epoch': 0.69} +{'loss': 1.9361, 'grad_norm': 2.2998454570770264, 'learning_rate': 4.307473598700244e-06, 'epoch': 0.69} +{'loss': 1.9503, 'grad_norm': 2.057570695877075, 'learning_rate': 4.305442729488221e-06, 'epoch': 0.69} +{'eval_loss': 2.0935096740722656, 'eval_runtime': 36.9124, 'eval_samples_per_second': 27.091, 'eval_steps_per_second': 1.138, 'epoch': 0.69} +{'loss': 1.8801, 'grad_norm': 2.1565167903900146, 'learning_rate': 4.303411860276198e-06, 'epoch': 0.69} +{'loss': 1.9861, 'grad_norm': 2.100191593170166, 'learning_rate': 4.301380991064176e-06, 'epoch': 0.69} +{'loss': 1.9588, 'grad_norm': 2.0314157009124756, 'learning_rate': 4.299350121852153e-06, 'epoch': 0.69} +{'loss': 1.9797, 'grad_norm': 2.270892381668091, 'learning_rate': 4.29731925264013e-06, 'epoch': 0.69} +{'loss': 1.9834, 'grad_norm': 1.9957224130630493, 'learning_rate': 4.2952883834281075e-06, 'epoch': 0.7} +{'loss': 1.9332, 'grad_norm': 2.145697593688965, 'learning_rate': 4.293257514216085e-06, 'epoch': 0.7} +{'loss': 1.8873, 'grad_norm': 2.3837900161743164, 'learning_rate': 4.291226645004062e-06, 'epoch': 0.7} +{'loss': 1.965, 'grad_norm': 1.9581835269927979, 'learning_rate': 4.289195775792039e-06, 'epoch': 0.7} +{'loss': 1.9812, 'grad_norm': 2.091428518295288, 'learning_rate': 4.287164906580017e-06, 'epoch': 0.7} +{'loss': 1.9201, 'grad_norm': 2.1069414615631104, 'learning_rate': 4.285134037367994e-06, 'epoch': 0.7} +{'eval_loss': 2.082728385925293, 'eval_runtime': 36.8994, 'eval_samples_per_second': 27.101, 'eval_steps_per_second': 1.138, 'epoch': 0.7} +{'loss': 1.9108, 'grad_norm': 1.9328324794769287, 'learning_rate': 4.283103168155971e-06, 'epoch': 0.7} +{'loss': 1.888, 'grad_norm': 2.0553345680236816, 'learning_rate': 4.281072298943948e-06, 'epoch': 0.7} +{'loss': 1.885, 'grad_norm': 2.153369426727295, 'learning_rate': 4.279041429731926e-06, 'epoch': 0.7} +{'loss': 1.9395, 'grad_norm': 2.1366844177246094, 'learning_rate': 4.277010560519903e-06, 'epoch': 0.71} +{'loss': 1.9104, 'grad_norm': 2.0654947757720947, 'learning_rate': 4.27497969130788e-06, 'epoch': 0.71} +{'loss': 1.9137, 'grad_norm': 2.058098077774048, 'learning_rate': 4.2729488220958575e-06, 'epoch': 0.71} +{'loss': 1.956, 'grad_norm': 2.2112984657287598, 'learning_rate': 4.270917952883835e-06, 'epoch': 0.71} +{'loss': 1.9758, 'grad_norm': 2.1838035583496094, 'learning_rate': 4.268887083671812e-06, 'epoch': 0.71} +{'loss': 1.8933, 'grad_norm': 2.142172336578369, 'learning_rate': 4.266856214459789e-06, 'epoch': 0.71} +{'loss': 2.0118, 'grad_norm': 2.149646520614624, 'learning_rate': 4.264825345247766e-06, 'epoch': 0.71} +{'eval_loss': 2.0800814628601074, 'eval_runtime': 36.8814, 'eval_samples_per_second': 27.114, 'eval_steps_per_second': 1.139, 'epoch': 0.71} +{'loss': 1.8852, 'grad_norm': 2.117799758911133, 'learning_rate': 4.262794476035744e-06, 'epoch': 0.71} +{'loss': 1.9385, 'grad_norm': 2.248161554336548, 'learning_rate': 4.26076360682372e-06, 'epoch': 0.71} +{'loss': 1.8479, 'grad_norm': 2.224853038787842, 'learning_rate': 4.2587327376116984e-06, 'epoch': 0.72} +{'loss': 1.9087, 'grad_norm': 2.2025816440582275, 'learning_rate': 4.256701868399675e-06, 'epoch': 0.72} +{'loss': 1.9438, 'grad_norm': 2.158379554748535, 'learning_rate': 4.254670999187653e-06, 'epoch': 0.72} +{'loss': 1.9343, 'grad_norm': 2.09287428855896, 'learning_rate': 4.252640129975629e-06, 'epoch': 0.72} +{'loss': 1.9391, 'grad_norm': 2.166685104370117, 'learning_rate': 4.2506092607636075e-06, 'epoch': 0.72} +{'loss': 1.9191, 'grad_norm': 2.125105142593384, 'learning_rate': 4.248578391551584e-06, 'epoch': 0.72} +{'loss': 1.862, 'grad_norm': 2.100684642791748, 'learning_rate': 4.246547522339562e-06, 'epoch': 0.72} +{'loss': 1.9038, 'grad_norm': 2.2361960411071777, 'learning_rate': 4.2445166531275385e-06, 'epoch': 0.72} +{'eval_loss': 2.0808706283569336, 'eval_runtime': 36.8185, 'eval_samples_per_second': 27.16, 'eval_steps_per_second': 1.141, 'epoch': 0.72} +{'loss': 1.9218, 'grad_norm': 2.309504270553589, 'learning_rate': 4.242485783915517e-06, 'epoch': 0.72} +{'loss': 1.8704, 'grad_norm': 2.102313280105591, 'learning_rate': 4.240454914703493e-06, 'epoch': 0.73} +{'loss': 1.8814, 'grad_norm': 2.2136611938476562, 'learning_rate': 4.23842404549147e-06, 'epoch': 0.73} +{'loss': 1.8775, 'grad_norm': 2.0127627849578857, 'learning_rate': 4.236393176279448e-06, 'epoch': 0.73} +{'loss': 1.8682, 'grad_norm': 2.192124366760254, 'learning_rate': 4.234362307067425e-06, 'epoch': 0.73} +{'loss': 1.9512, 'grad_norm': 2.4385814666748047, 'learning_rate': 4.232331437855402e-06, 'epoch': 0.73} +{'loss': 1.9669, 'grad_norm': 2.0868494510650635, 'learning_rate': 4.2303005686433795e-06, 'epoch': 0.73} +{'loss': 1.971, 'grad_norm': 2.2664904594421387, 'learning_rate': 4.228269699431357e-06, 'epoch': 0.73} +{'loss': 1.9259, 'grad_norm': 2.043488025665283, 'learning_rate': 4.226238830219334e-06, 'epoch': 0.73} +{'loss': 1.9357, 'grad_norm': 2.0816924571990967, 'learning_rate': 4.224207961007311e-06, 'epoch': 0.73} +{'eval_loss': 2.0858654975891113, 'eval_runtime': 36.7872, 'eval_samples_per_second': 27.183, 'eval_steps_per_second': 1.142, 'epoch': 0.73} +{'loss': 1.9234, 'grad_norm': 2.134960889816284, 'learning_rate': 4.2221770917952886e-06, 'epoch': 0.74} +{'loss': 1.9128, 'grad_norm': 2.1693387031555176, 'learning_rate': 4.220146222583266e-06, 'epoch': 0.74} +{'loss': 1.9364, 'grad_norm': 2.2874763011932373, 'learning_rate': 4.218115353371243e-06, 'epoch': 0.74} +{'loss': 1.9281, 'grad_norm': 2.123231887817383, 'learning_rate': 4.21608448415922e-06, 'epoch': 0.74} +{'loss': 1.8954, 'grad_norm': 2.020890712738037, 'learning_rate': 4.214053614947198e-06, 'epoch': 0.74} +{'loss': 1.9805, 'grad_norm': 2.0817489624023438, 'learning_rate': 4.212022745735175e-06, 'epoch': 0.74} +{'loss': 1.8726, 'grad_norm': 2.266101360321045, 'learning_rate': 4.209991876523152e-06, 'epoch': 0.74} +{'loss': 1.9528, 'grad_norm': 2.0021142959594727, 'learning_rate': 4.2079610073111295e-06, 'epoch': 0.74} +{'loss': 1.8749, 'grad_norm': 2.213418483734131, 'learning_rate': 4.205930138099107e-06, 'epoch': 0.74} +{'loss': 1.9787, 'grad_norm': 2.2187159061431885, 'learning_rate': 4.203899268887084e-06, 'epoch': 0.75} +{'eval_loss': 2.084170341491699, 'eval_runtime': 36.7678, 'eval_samples_per_second': 27.198, 'eval_steps_per_second': 1.142, 'epoch': 0.75} +{'loss': 1.9257, 'grad_norm': 1.986974835395813, 'learning_rate': 4.201868399675061e-06, 'epoch': 0.75} +{'loss': 1.9743, 'grad_norm': 2.0920112133026123, 'learning_rate': 4.199837530463039e-06, 'epoch': 0.75} +{'loss': 1.8572, 'grad_norm': 2.2249104976654053, 'learning_rate': 4.197806661251016e-06, 'epoch': 0.75} +{'loss': 1.9271, 'grad_norm': 2.0482609272003174, 'learning_rate': 4.195775792038993e-06, 'epoch': 0.75} +{'loss': 1.9097, 'grad_norm': 2.1768155097961426, 'learning_rate': 4.19374492282697e-06, 'epoch': 0.75} +{'loss': 1.8393, 'grad_norm': 2.1729984283447266, 'learning_rate': 4.191714053614948e-06, 'epoch': 0.75} +{'loss': 1.9426, 'grad_norm': 2.2250611782073975, 'learning_rate': 4.189683184402924e-06, 'epoch': 0.75} +{'loss': 1.8935, 'grad_norm': 2.1344521045684814, 'learning_rate': 4.187652315190902e-06, 'epoch': 0.75} +{'loss': 1.9631, 'grad_norm': 2.2652342319488525, 'learning_rate': 4.185621445978879e-06, 'epoch': 0.76} +{'loss': 1.8911, 'grad_norm': 2.2338240146636963, 'learning_rate': 4.183590576766857e-06, 'epoch': 0.76} +{'eval_loss': 2.081585168838501, 'eval_runtime': 36.9744, 'eval_samples_per_second': 27.046, 'eval_steps_per_second': 1.136, 'epoch': 0.76} +{'loss': 1.9124, 'grad_norm': 2.099368095397949, 'learning_rate': 4.181559707554833e-06, 'epoch': 0.76} +{'loss': 1.9653, 'grad_norm': 2.2774770259857178, 'learning_rate': 4.179528838342811e-06, 'epoch': 0.76} +{'loss': 1.883, 'grad_norm': 2.1390492916107178, 'learning_rate': 4.177497969130788e-06, 'epoch': 0.76} +{'loss': 1.8933, 'grad_norm': 2.1724212169647217, 'learning_rate': 4.175467099918766e-06, 'epoch': 0.76} +{'loss': 1.9805, 'grad_norm': 2.0856740474700928, 'learning_rate': 4.173436230706742e-06, 'epoch': 0.76} +{'loss': 1.9129, 'grad_norm': 2.1491377353668213, 'learning_rate': 4.1714053614947205e-06, 'epoch': 0.76} +{'loss': 1.8939, 'grad_norm': 2.207265615463257, 'learning_rate': 4.169374492282697e-06, 'epoch': 0.76} +{'loss': 1.9003, 'grad_norm': 2.091902494430542, 'learning_rate': 4.167343623070675e-06, 'epoch': 0.76} +{'loss': 1.9565, 'grad_norm': 2.048015594482422, 'learning_rate': 4.165312753858652e-06, 'epoch': 0.77} +{'loss': 1.9066, 'grad_norm': 2.1699411869049072, 'learning_rate': 4.163281884646629e-06, 'epoch': 0.77} +{'eval_loss': 2.076061248779297, 'eval_runtime': 37.077, 'eval_samples_per_second': 26.971, 'eval_steps_per_second': 1.133, 'epoch': 0.77} +{'loss': 1.9024, 'grad_norm': 2.1105830669403076, 'learning_rate': 4.161251015434607e-06, 'epoch': 0.77} +{'loss': 1.9268, 'grad_norm': 2.0144476890563965, 'learning_rate': 4.159220146222583e-06, 'epoch': 0.77} +{'loss': 1.879, 'grad_norm': 2.060319662094116, 'learning_rate': 4.157189277010561e-06, 'epoch': 0.77} +{'loss': 1.9073, 'grad_norm': 2.14819598197937, 'learning_rate': 4.155158407798538e-06, 'epoch': 0.77} +{'loss': 1.947, 'grad_norm': 2.2959697246551514, 'learning_rate': 4.153127538586516e-06, 'epoch': 0.77} +{'loss': 1.9127, 'grad_norm': 2.1901297569274902, 'learning_rate': 4.151096669374492e-06, 'epoch': 0.77} +{'loss': 1.9376, 'grad_norm': 2.264690637588501, 'learning_rate': 4.1490658001624705e-06, 'epoch': 0.77} +{'loss': 1.9048, 'grad_norm': 2.155043601989746, 'learning_rate': 4.147034930950447e-06, 'epoch': 0.78} +{'loss': 1.9049, 'grad_norm': 2.059197425842285, 'learning_rate': 4.145004061738425e-06, 'epoch': 0.78} +{'loss': 1.9038, 'grad_norm': 2.4279260635375977, 'learning_rate': 4.1429731925264015e-06, 'epoch': 0.78} +{'eval_loss': 2.074542999267578, 'eval_runtime': 36.8289, 'eval_samples_per_second': 27.153, 'eval_steps_per_second': 1.14, 'epoch': 0.78} +{'loss': 1.8325, 'grad_norm': 2.1462533473968506, 'learning_rate': 4.140942323314379e-06, 'epoch': 0.78} +{'loss': 1.8739, 'grad_norm': 2.3161439895629883, 'learning_rate': 4.138911454102356e-06, 'epoch': 0.78} +{'loss': 1.8985, 'grad_norm': 2.130906343460083, 'learning_rate': 4.136880584890333e-06, 'epoch': 0.78} +{'loss': 1.9336, 'grad_norm': 2.107179641723633, 'learning_rate': 4.1348497156783106e-06, 'epoch': 0.78} +{'loss': 1.9318, 'grad_norm': 2.4675867557525635, 'learning_rate': 4.132818846466288e-06, 'epoch': 0.78} +{'loss': 1.9279, 'grad_norm': 2.1309103965759277, 'learning_rate': 4.130787977254265e-06, 'epoch': 0.78} +{'loss': 1.9428, 'grad_norm': 2.2152087688446045, 'learning_rate': 4.128757108042242e-06, 'epoch': 0.79} +{'loss': 1.9456, 'grad_norm': 2.02030873298645, 'learning_rate': 4.12672623883022e-06, 'epoch': 0.79} +{'loss': 1.9994, 'grad_norm': 2.2501220703125, 'learning_rate': 4.124695369618197e-06, 'epoch': 0.79} +{'loss': 1.9181, 'grad_norm': 2.082099199295044, 'learning_rate': 4.122664500406174e-06, 'epoch': 0.79} +{'eval_loss': 2.071835994720459, 'eval_runtime': 36.8879, 'eval_samples_per_second': 27.109, 'eval_steps_per_second': 1.139, 'epoch': 0.79} +{'loss': 1.9112, 'grad_norm': 2.121677875518799, 'learning_rate': 4.1206336311941515e-06, 'epoch': 0.79} +{'loss': 1.9328, 'grad_norm': 2.154696226119995, 'learning_rate': 4.118602761982129e-06, 'epoch': 0.79} +{'loss': 1.8682, 'grad_norm': 2.141899347305298, 'learning_rate': 4.116571892770106e-06, 'epoch': 0.79} +{'loss': 1.85, 'grad_norm': 2.1390345096588135, 'learning_rate': 4.114541023558083e-06, 'epoch': 0.79} +{'loss': 1.9754, 'grad_norm': 2.22886061668396, 'learning_rate': 4.112510154346061e-06, 'epoch': 0.79} +{'loss': 1.8985, 'grad_norm': 2.2299511432647705, 'learning_rate': 4.110479285134038e-06, 'epoch': 0.8} +{'loss': 1.918, 'grad_norm': 2.251913070678711, 'learning_rate': 4.108448415922015e-06, 'epoch': 0.8} +{'loss': 1.9469, 'grad_norm': 2.2145745754241943, 'learning_rate': 4.1064175467099924e-06, 'epoch': 0.8} +{'loss': 1.9395, 'grad_norm': 2.0919570922851562, 'learning_rate': 4.10438667749797e-06, 'epoch': 0.8} +{'loss': 1.7958, 'grad_norm': 2.3385298252105713, 'learning_rate': 4.102355808285947e-06, 'epoch': 0.8} +{'eval_loss': 2.0747177600860596, 'eval_runtime': 36.894, 'eval_samples_per_second': 27.105, 'eval_steps_per_second': 1.138, 'epoch': 0.8} +{'loss': 1.9948, 'grad_norm': 2.3434054851531982, 'learning_rate': 4.100324939073924e-06, 'epoch': 0.8} +{'loss': 1.8994, 'grad_norm': 2.129086971282959, 'learning_rate': 4.0982940698619015e-06, 'epoch': 0.8} +{'loss': 1.8951, 'grad_norm': 2.2564284801483154, 'learning_rate': 4.096263200649879e-06, 'epoch': 0.8} +{'loss': 1.9083, 'grad_norm': 2.53267240524292, 'learning_rate': 4.094232331437856e-06, 'epoch': 0.8} +{'loss': 1.8775, 'grad_norm': 2.164807081222534, 'learning_rate': 4.092201462225833e-06, 'epoch': 0.81} +{'loss': 1.8777, 'grad_norm': 2.3593950271606445, 'learning_rate': 4.090170593013811e-06, 'epoch': 0.81} +{'loss': 1.9196, 'grad_norm': 2.470942258834839, 'learning_rate': 4.088139723801787e-06, 'epoch': 0.81} +{'loss': 1.86, 'grad_norm': 2.0244338512420654, 'learning_rate': 4.086108854589765e-06, 'epoch': 0.81} +{'loss': 1.8832, 'grad_norm': 2.2432093620300293, 'learning_rate': 4.084077985377742e-06, 'epoch': 0.81} +{'loss': 1.882, 'grad_norm': 2.087015390396118, 'learning_rate': 4.08204711616572e-06, 'epoch': 0.81} +{'eval_loss': 2.0757997035980225, 'eval_runtime': 36.8834, 'eval_samples_per_second': 27.112, 'eval_steps_per_second': 1.139, 'epoch': 0.81} +{'loss': 1.9526, 'grad_norm': 2.038623571395874, 'learning_rate': 4.080016246953696e-06, 'epoch': 0.81} +{'loss': 1.8739, 'grad_norm': 2.1820836067199707, 'learning_rate': 4.077985377741674e-06, 'epoch': 0.81} +{'loss': 1.941, 'grad_norm': 2.1047675609588623, 'learning_rate': 4.075954508529651e-06, 'epoch': 0.81} +{'loss': 1.9522, 'grad_norm': 2.1180074214935303, 'learning_rate': 4.073923639317629e-06, 'epoch': 0.82} +{'loss': 1.9644, 'grad_norm': 2.1034464836120605, 'learning_rate': 4.071892770105605e-06, 'epoch': 0.82} +{'loss': 1.8953, 'grad_norm': 2.1570026874542236, 'learning_rate': 4.069861900893583e-06, 'epoch': 0.82} +{'loss': 1.8557, 'grad_norm': 2.041189432144165, 'learning_rate': 4.06783103168156e-06, 'epoch': 0.82} +{'loss': 1.8746, 'grad_norm': 2.1053237915039062, 'learning_rate': 4.065800162469538e-06, 'epoch': 0.82} +{'loss': 1.8773, 'grad_norm': 2.223841667175293, 'learning_rate': 4.063769293257514e-06, 'epoch': 0.82} +{'loss': 1.9193, 'grad_norm': 2.2014336585998535, 'learning_rate': 4.061738424045492e-06, 'epoch': 0.82} +{'eval_loss': 2.078728437423706, 'eval_runtime': 36.8539, 'eval_samples_per_second': 27.134, 'eval_steps_per_second': 1.14, 'epoch': 0.82} diff --git a/wandb/run-20241105_163244-ax7d1x4c/files/requirements.txt b/wandb/run-20241105_163244-ax7d1x4c/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..95a931302e269cc9e4fa5b719b6511f176ee2416 --- /dev/null +++ b/wandb/run-20241105_163244-ax7d1x4c/files/requirements.txt @@ -0,0 +1,147 @@ +funcsigs==1.0.2 +sentry-sdk==2.17.0 +multiprocess==0.70.16 +numpy==1.26.2 +pluralizer==1.2.0 +debugpy==1.6.7 +nvidia-cudnn-cu11==8.5.0.96 +deepspeed==0.15.2 +data==0.4 +pandas==2.1.3 +tomli==2.0.1 +charset-normalizer==3.3.2 +attrs==24.2.0 +aiosignal==1.3.1 +fsspec==2023.10.0 +nvidia-cusparse-cu11==11.7.4.91 +zipp==3.12.0 +mypy-extensions==1.0.0 +datasets==3.0.1 +joblib==1.3.2 +hjson==3.1.0 +traitlets==5.7.1 +stack-data==0.6.0 +transformers==4.45.1 +sympy==1.11.1 +Pygments==2.15.0 +docker-pycreds==0.4.0 +dill==0.3.8 +wheel==0.44.0 +prompt-toolkit==3.0.30 +parso==0.8.3 +ipykernel==6.23.1 +pyarrow==17.0.0 +certifi==2023.11.17 +nvidia-cufft-cu11==10.9.0.58 +six==1.16.0 +pydantic==2.9.2 +click==8.1.7 +nest-asyncio==1.5.6 +gmpy2==2.1.0 +matplotlib==3.8.2 +scipy==1.11.4 +typing_extensions==4.12.2 +statsmodels==0.14.0 +huggingface-hub==0.25.0 +frozenlist==1.4.1 +gpustat==1.1.1 +nvidia-nvtx-cu11==11.7.91 +safetensors==0.4.5 +stanza==1.9.2 +decorator==5.1.1 +seaborn==0.13.0 +sentencepiece==0.2.0 +PyYAML==6.0.1 +black==24.8.0 +protobuf==4.25.1 +pickleshare==0.7.5 +peft==0.13.0 +triton==2.0.0 +nvidia-cuda-runtime-cu11==11.7.99 +Jinja2==3.1.2 +nvidia-cusolver-cu11==11.4.0.1 +executing==1.2.0 +jupyter_client==8.1.0 +pluggy==1.3.0 +cmake==3.30.3 +pytz==2023.3.post1 +aiohappyeyeballs==2.4.2 +kiwisolver==1.4.5 +py-cpuinfo==9.0.0 +Pillow==10.1.0 +ptyprocess==0.7.0 +importlib_resources==6.4.5 +GitPython==3.1.43 +importlib-metadata==6.0.0 +iniconfig==2.0.0 +scikit-learn==1.3.2 +exceptiongroup==1.1.0 +networkx==2.8.6 +accelerate==1.0.0 +nltk==3.8.1 +shutilwhich==1.1.0 +fonttools==4.45.1 +future==0.18.3 +aiohttp==3.10.6 +wcwidth==0.2.5 +idna==3.6 +filelock==3.12.2 +pathspec==0.12.1 +jupyter_core==5.1.0 +lit==18.1.8 +nvidia-curand-cu11==10.2.10.91 +nvidia-cublas-cu11==11.10.3.66 +nvidia-ml-py==12.560.30 +msgpack==1.1.0 +python-dateutil==2.8.2 +blessed==1.20.0 +packaging==23.0 +gitdb==4.0.11 +yarl==1.13.0 +emoji==2.8.0 +tzdata==2023.3 +cycler==0.12.1 +tornado==6.2 +backcall==0.2.0 +plotnine==0.12.4 +ninja==1.11.1.1 +latex==0.7.0 +wandb==0.18.5 +setproctitle==1.3.3 +threadpoolctl==3.2.0 +requests==2.32.3 +pyparsing==3.1.1 +smmap==5.0.1 +pyzmq==23.0.0 +async-timeout==4.0.3 +annotated-types==0.7.0 +matplotlib-inline==0.1.6 +latexcodec==1.0.0 +ipython==8.0.0 +patsy==0.5.3 +contourpy==1.2.0 +multidict==6.1.0 +mizani==0.9.3 +urllib3==2.1.0 +tokenizers==0.20.0 +MarkupSafe==2.1.2 +pip==24.2 +pexpect==4.8.0 +tqdm==4.66.5 +jedi==0.18.2 +pydantic_core==2.23.4 +tempdir==0.7.1 +mpmath==1.2.1 +setuptools==72.1.0 +pytest==7.4.3 +pure-eval==0.2.2 +psutil==5.9.1 +comm==0.1.2 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +regex==2023.10.3 +platformdirs==2.5.2 +asttokens==2.2.1 +torch==2.0.0 +nvidia-nccl-cu11==2.14.3 +xxhash==3.5.0 diff --git a/wandb/run-20241105_163244-ax7d1x4c/files/wandb-metadata.json b/wandb/run-20241105_163244-ax7d1x4c/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..4bdbbb8508429327633e36f22976ad6f0dccd7b2 --- /dev/null +++ b/wandb/run-20241105_163244-ax7d1x4c/files/wandb-metadata.json @@ -0,0 +1,97 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-11-05T21:32:44.682105Z", + "args": [ + "--perturbation", + "shuffle_deterministic57", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "3", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", + "codePath": "train/train_deep_wandb.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_deep_wandb.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA RTX A6000", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888559353856", + "used": "1785985114112" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + } + ], + "cudaVersion": "11.8" +} \ No newline at end of file diff --git a/wandb/run-20241105_163244-ax7d1x4c/logs/debug-internal.log b/wandb/run-20241105_163244-ax7d1x4c/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..c1cb75acff95595ab14ca036a3662b6ecfed46b4 --- /dev/null +++ b/wandb/run-20241105_163244-ax7d1x4c/logs/debug-internal.log @@ -0,0 +1,8 @@ +{"time":"2024-11-05T16:32:44.684204084-05:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-11-05T16:32:44.684214704-05:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_163244-ax7d1x4c/logs/debug-core.log"} +{"time":"2024-11-05T16:32:44.791124607-05:00","level":"INFO","msg":"created new stream","id":"ax7d1x4c"} +{"time":"2024-11-05T16:32:44.791159098-05:00","level":"INFO","msg":"stream: started","id":"ax7d1x4c"} +{"time":"2024-11-05T16:32:44.791187858-05:00","level":"INFO","msg":"sender: started","stream_id":"ax7d1x4c"} +{"time":"2024-11-05T16:32:44.791184658-05:00","level":"INFO","msg":"handler: started","stream_id":{"value":"ax7d1x4c"}} +{"time":"2024-11-05T16:32:44.791180678-05:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"ax7d1x4c"}} +{"time":"2024-11-05T16:32:45.03367425-05:00","level":"INFO","msg":"Starting system monitor"} diff --git a/wandb/run-20241105_163244-ax7d1x4c/logs/debug.log b/wandb/run-20241105_163244-ax7d1x4c/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..206e83e113f433b4b875cc73aa0bc9a63db00744 --- /dev/null +++ b/wandb/run-20241105_163244-ax7d1x4c/logs/debug.log @@ -0,0 +1,29 @@ +2024-11-05 16:32:44,680 INFO MainThread:1780849 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-11-05 16:32:44,680 INFO MainThread:1780849 [wandb_setup.py:_flush():79] Configure stats pid to 1780849 +2024-11-05 16:32:44,680 INFO MainThread:1780849 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-11-05 16:32:44,680 INFO MainThread:1780849 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-11-05 16:32:44,680 INFO MainThread:1780849 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-05 16:32:44,680 INFO MainThread:1780849 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-05 16:32:44,680 INFO MainThread:1780849 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'} +2024-11-05 16:32:44,680 INFO MainThread:1780849 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-05 16:32:44,680 INFO MainThread:1780849 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_163244-ax7d1x4c/logs/debug.log +2024-11-05 16:32:44,680 INFO MainThread:1780849 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_163244-ax7d1x4c/logs/debug-internal.log +2024-11-05 16:32:44,680 INFO MainThread:1780849 [wandb_init.py:init():621] calling init triggers +2024-11-05 16:32:44,680 INFO MainThread:1780849 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-11-05 16:32:44,680 INFO MainThread:1780849 [wandb_init.py:init():671] starting backend +2024-11-05 16:32:44,680 INFO MainThread:1780849 [wandb_init.py:init():675] sending inform_init request +2024-11-05 16:32:44,681 INFO MainThread:1780849 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-05 16:32:44,681 INFO MainThread:1780849 [wandb_init.py:init():688] backend started and connected +2024-11-05 16:32:44,684 INFO MainThread:1780849 [wandb_init.py:init():783] updated telemetry +2024-11-05 16:32:44,723 INFO MainThread:1780849 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-11-05 16:32:45,029 INFO MainThread:1780849 [wandb_init.py:init():867] starting run threads in backend +2024-11-05 16:32:45,118 INFO MainThread:1780849 [wandb_run.py:_console_start():2463] atexit reg +2024-11-05 16:32:45,118 INFO MainThread:1780849 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-11-05 16:32:45,118 INFO MainThread:1780849 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-11-05 16:32:45,118 INFO MainThread:1780849 [wandb_run.py:_redirect():2401] Redirects installed. +2024-11-05 16:32:45,120 INFO MainThread:1780849 [wandb_init.py:init():911] run started, returning control to user process +2024-11-05 16:32:45,120 INFO MainThread:1780849 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_deterministic57', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0, 'lr': 5e-06} +2024-11-05 16:36:12,718 INFO MainThread:1780849 [wandb_run.py:_config_callback():1390] config_cb None None {'vocab_size': 128256, 'max_position_embeddings': 131072, 'hidden_size': 3072, 'intermediate_size': 8192, 'num_hidden_layers': 28, 'num_attention_heads': 24, 'num_key_value_heads': 8, 'hidden_act': 'silu', 'initializer_range': 0.02, 'rms_norm_eps': 1e-05, 'pretraining_tp': 1, 'use_cache': True, 'rope_theta': 500000.0, 'rope_scaling': {'factor': 32.0, 'high_freq_factor': 4.0, 'low_freq_factor': 1.0, 'original_max_position_embeddings': 8192, 'rope_type': 'llama3'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 128, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'bfloat16', 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': True, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['LlamaForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 128000, 'pad_token_id': None, 'eos_token_id': 128001, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'meta-llama/Llama-3.2-3B', 'transformers_version': '4.45.1', 'model_type': 'llama', 'output_dir': './checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic57_10M_seed0/runs', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': True, 'do_predict': False, 'eval_strategy': 'steps', 'prediction_loss_only': False, 'per_device_train_batch_size': 3, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 2, 'eval_accumulation_steps': None, 'eval_delay': 0, 'torch_empty_cache_steps': None, 'learning_rate': 5e-06, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 3, 'max_steps': -1, 'lr_scheduler_type': 'linear', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.1, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': './logs', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 1, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 100, 'save_total_limit': None, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 0, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': False, 'fp16': True, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 10, 'dataloader_num_workers': 0, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': './checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic57_10M_seed0/runs', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': 'deepspeed_config/train_dp_config.json', 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'evaluation_strategy': 'steps', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False, 'use_liger_kernel': False, 'eval_use_gather_object': False} +2024-11-05 16:36:12,720 INFO MainThread:1780849 [wandb_config.py:__setitem__():154] config set model/num_parameters = 3212749824 - > +2024-11-05 16:36:12,720 INFO MainThread:1780849 [wandb_run.py:_config_callback():1390] config_cb model/num_parameters 3212749824 None diff --git a/wandb/run-20241105_163248-ec3mgg2q/files/output.log b/wandb/run-20241105_163248-ec3mgg2q/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..26f07e0232149a54351de21407c3c5f3e64bbccd --- /dev/null +++ b/wandb/run-20241105_163248-ec3mgg2q/files/output.log @@ -0,0 +1,19 @@ +Downloading shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [01:13<00:00, 36.71s/it] +Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:05<00:00, 2.91s/it] +generation_config.json: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 185/185 [00:00<00:00, 33.5kB/s] +tokenized_valid: Dataset({ + features: ['input_ids', 'attention_mask'], + num_rows: 1000 +}) +/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +[2024-11-05 16:34:10,038] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-11-05 16:34:18,703] [INFO] [comm.py:652:init_distributed] cdb=None +Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination +Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root... +Emitting ninja build file /home/chunhui/.cache/torch_extensions/py39_cu117/cpu_adam/build.ninja... +Building extension module cpu_adam... +Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +Loading extension module cpu_adam... +Time to load cpu_adam op: 5.242245674133301 seconds +wandb: WARNING Fatal error while uploading data. Some run data will not be synced, but it will still be written to disk. Use `wandb sync` at the end of the run to try uploading. diff --git a/wandb/run-20241105_163248-ec3mgg2q/files/requirements.txt b/wandb/run-20241105_163248-ec3mgg2q/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..95a931302e269cc9e4fa5b719b6511f176ee2416 --- /dev/null +++ b/wandb/run-20241105_163248-ec3mgg2q/files/requirements.txt @@ -0,0 +1,147 @@ +funcsigs==1.0.2 +sentry-sdk==2.17.0 +multiprocess==0.70.16 +numpy==1.26.2 +pluralizer==1.2.0 +debugpy==1.6.7 +nvidia-cudnn-cu11==8.5.0.96 +deepspeed==0.15.2 +data==0.4 +pandas==2.1.3 +tomli==2.0.1 +charset-normalizer==3.3.2 +attrs==24.2.0 +aiosignal==1.3.1 +fsspec==2023.10.0 +nvidia-cusparse-cu11==11.7.4.91 +zipp==3.12.0 +mypy-extensions==1.0.0 +datasets==3.0.1 +joblib==1.3.2 +hjson==3.1.0 +traitlets==5.7.1 +stack-data==0.6.0 +transformers==4.45.1 +sympy==1.11.1 +Pygments==2.15.0 +docker-pycreds==0.4.0 +dill==0.3.8 +wheel==0.44.0 +prompt-toolkit==3.0.30 +parso==0.8.3 +ipykernel==6.23.1 +pyarrow==17.0.0 +certifi==2023.11.17 +nvidia-cufft-cu11==10.9.0.58 +six==1.16.0 +pydantic==2.9.2 +click==8.1.7 +nest-asyncio==1.5.6 +gmpy2==2.1.0 +matplotlib==3.8.2 +scipy==1.11.4 +typing_extensions==4.12.2 +statsmodels==0.14.0 +huggingface-hub==0.25.0 +frozenlist==1.4.1 +gpustat==1.1.1 +nvidia-nvtx-cu11==11.7.91 +safetensors==0.4.5 +stanza==1.9.2 +decorator==5.1.1 +seaborn==0.13.0 +sentencepiece==0.2.0 +PyYAML==6.0.1 +black==24.8.0 +protobuf==4.25.1 +pickleshare==0.7.5 +peft==0.13.0 +triton==2.0.0 +nvidia-cuda-runtime-cu11==11.7.99 +Jinja2==3.1.2 +nvidia-cusolver-cu11==11.4.0.1 +executing==1.2.0 +jupyter_client==8.1.0 +pluggy==1.3.0 +cmake==3.30.3 +pytz==2023.3.post1 +aiohappyeyeballs==2.4.2 +kiwisolver==1.4.5 +py-cpuinfo==9.0.0 +Pillow==10.1.0 +ptyprocess==0.7.0 +importlib_resources==6.4.5 +GitPython==3.1.43 +importlib-metadata==6.0.0 +iniconfig==2.0.0 +scikit-learn==1.3.2 +exceptiongroup==1.1.0 +networkx==2.8.6 +accelerate==1.0.0 +nltk==3.8.1 +shutilwhich==1.1.0 +fonttools==4.45.1 +future==0.18.3 +aiohttp==3.10.6 +wcwidth==0.2.5 +idna==3.6 +filelock==3.12.2 +pathspec==0.12.1 +jupyter_core==5.1.0 +lit==18.1.8 +nvidia-curand-cu11==10.2.10.91 +nvidia-cublas-cu11==11.10.3.66 +nvidia-ml-py==12.560.30 +msgpack==1.1.0 +python-dateutil==2.8.2 +blessed==1.20.0 +packaging==23.0 +gitdb==4.0.11 +yarl==1.13.0 +emoji==2.8.0 +tzdata==2023.3 +cycler==0.12.1 +tornado==6.2 +backcall==0.2.0 +plotnine==0.12.4 +ninja==1.11.1.1 +latex==0.7.0 +wandb==0.18.5 +setproctitle==1.3.3 +threadpoolctl==3.2.0 +requests==2.32.3 +pyparsing==3.1.1 +smmap==5.0.1 +pyzmq==23.0.0 +async-timeout==4.0.3 +annotated-types==0.7.0 +matplotlib-inline==0.1.6 +latexcodec==1.0.0 +ipython==8.0.0 +patsy==0.5.3 +contourpy==1.2.0 +multidict==6.1.0 +mizani==0.9.3 +urllib3==2.1.0 +tokenizers==0.20.0 +MarkupSafe==2.1.2 +pip==24.2 +pexpect==4.8.0 +tqdm==4.66.5 +jedi==0.18.2 +pydantic_core==2.23.4 +tempdir==0.7.1 +mpmath==1.2.1 +setuptools==72.1.0 +pytest==7.4.3 +pure-eval==0.2.2 +psutil==5.9.1 +comm==0.1.2 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +regex==2023.10.3 +platformdirs==2.5.2 +asttokens==2.2.1 +torch==2.0.0 +nvidia-nccl-cu11==2.14.3 +xxhash==3.5.0 diff --git a/wandb/run-20241105_163248-ec3mgg2q/files/wandb-metadata.json b/wandb/run-20241105_163248-ec3mgg2q/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..373546eea41e1fa2d3b2350f1e2b62879555776d --- /dev/null +++ b/wandb/run-20241105_163248-ec3mgg2q/files/wandb-metadata.json @@ -0,0 +1,97 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-11-05T21:32:48.545994Z", + "args": [ + "--perturbation", + "shuffle_deterministic21", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "3", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", + "codePath": "train/train_deep_wandb.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_deep_wandb.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA RTX A6000", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888559353856", + "used": "1785985146880" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + } + ], + "cudaVersion": "11.8" +} \ No newline at end of file diff --git a/wandb/run-20241105_163248-ec3mgg2q/logs/debug-internal.log b/wandb/run-20241105_163248-ec3mgg2q/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..614360238eb9ac4caf415bca1952b80a6f90cf53 --- /dev/null +++ b/wandb/run-20241105_163248-ec3mgg2q/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2024-11-05T16:32:48.548416625-05:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-11-05T16:32:48.548430775-05:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_163248-ec3mgg2q/logs/debug-core.log"} +{"time":"2024-11-05T16:32:48.655280611-05:00","level":"INFO","msg":"created new stream","id":"ec3mgg2q"} +{"time":"2024-11-05T16:32:48.655331591-05:00","level":"INFO","msg":"stream: started","id":"ec3mgg2q"} +{"time":"2024-11-05T16:32:48.655751403-05:00","level":"INFO","msg":"sender: started","stream_id":"ec3mgg2q"} +{"time":"2024-11-05T16:32:48.655730723-05:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"ec3mgg2q"}} +{"time":"2024-11-05T16:32:48.655791383-05:00","level":"INFO","msg":"handler: started","stream_id":{"value":"ec3mgg2q"}} +{"time":"2024-11-05T16:32:48.897702177-05:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-11-05T20:13:02.066334807-05:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"} +{"time":"2024-11-05T22:36:23.769093745-05:00","level":"ERROR","msg":"HTTP error","status":404,"method":"POST","url":"https://api.wandb.ai/files/yaning1001-dartmouth-college/exp-impo-shuffle/ec3mgg2q/file_stream"} +{"time":"2024-11-05T22:36:23.773511257-05:00","level":"ERROR+4","msg":"filestream: fatal error: filestream: failed to upload: 404 Not Found path=files/yaning1001-dartmouth-college/exp-impo-shuffle/ec3mgg2q/file_stream: {\"error\":\"project yaning1001-dartmouth-college/exp-impo-shuffle not found while streaming file\"}"} diff --git a/wandb/run-20241105_163248-ec3mgg2q/logs/debug.log b/wandb/run-20241105_163248-ec3mgg2q/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..01b952c0a10a9f4462bb1715947a69658b96be14 --- /dev/null +++ b/wandb/run-20241105_163248-ec3mgg2q/logs/debug.log @@ -0,0 +1,26 @@ +2024-11-05 16:32:48,542 INFO MainThread:1780934 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-11-05 16:32:48,542 INFO MainThread:1780934 [wandb_setup.py:_flush():79] Configure stats pid to 1780934 +2024-11-05 16:32:48,543 INFO MainThread:1780934 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-11-05 16:32:48,543 INFO MainThread:1780934 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-11-05 16:32:48,543 INFO MainThread:1780934 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-05 16:32:48,543 INFO MainThread:1780934 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-05 16:32:48,543 INFO MainThread:1780934 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'} +2024-11-05 16:32:48,543 INFO MainThread:1780934 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-05 16:32:48,543 INFO MainThread:1780934 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_163248-ec3mgg2q/logs/debug.log +2024-11-05 16:32:48,543 INFO MainThread:1780934 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_163248-ec3mgg2q/logs/debug-internal.log +2024-11-05 16:32:48,543 INFO MainThread:1780934 [wandb_init.py:init():621] calling init triggers +2024-11-05 16:32:48,543 INFO MainThread:1780934 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-11-05 16:32:48,543 INFO MainThread:1780934 [wandb_init.py:init():671] starting backend +2024-11-05 16:32:48,543 INFO MainThread:1780934 [wandb_init.py:init():675] sending inform_init request +2024-11-05 16:32:48,545 INFO MainThread:1780934 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-05 16:32:48,545 INFO MainThread:1780934 [wandb_init.py:init():688] backend started and connected +2024-11-05 16:32:48,549 INFO MainThread:1780934 [wandb_init.py:init():783] updated telemetry +2024-11-05 16:32:48,575 INFO MainThread:1780934 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-11-05 16:32:48,894 INFO MainThread:1780934 [wandb_init.py:init():867] starting run threads in backend +2024-11-05 16:32:48,983 INFO MainThread:1780934 [wandb_run.py:_console_start():2463] atexit reg +2024-11-05 16:32:48,983 INFO MainThread:1780934 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-11-05 16:32:48,983 INFO MainThread:1780934 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-11-05 16:32:48,983 INFO MainThread:1780934 [wandb_run.py:_redirect():2401] Redirects installed. +2024-11-05 16:32:48,985 INFO MainThread:1780934 [wandb_init.py:init():911] run started, returning control to user process +2024-11-05 16:32:48,985 INFO MainThread:1780934 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_deterministic21', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0, 'lr': 5e-06} diff --git a/wandb/run-20241113_180230-zrr91ih2/files/config.yaml b/wandb/run-20241113_180230-zrr91ih2/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..164653596defa3a070943f5b9f7cd18e7f2d95cc --- /dev/null +++ b/wandb/run-20241113_180230-zrr91ih2/files/config.yaml @@ -0,0 +1,50 @@ +_wandb: + value: + cli_version: 0.18.5 + m: [] + python_version: 3.9.19 + t: + "1": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "2": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "3": + - 2 + - 13 + - 23 + - 55 + "4": 3.9.19 + "5": 0.18.5 + "6": 4.45.1 + "8": + - 5 + "12": 0.18.5 + "13": linux-x86_64 +batch_size: + value: 3 +epoch: + value: 3 +lr: + value: 5e-06 +perturbation: + value: reverse_partial +seed: + value: 0 +train_set: + value: 10M diff --git a/wandb/run-20241113_180230-zrr91ih2/files/output.log b/wandb/run-20241113_180230-zrr91ih2/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..73bff535e797da4c55b23a654055300d6bd1b5e1 --- /dev/null +++ b/wandb/run-20241113_180230-zrr91ih2/files/output.log @@ -0,0 +1,22 @@ +model-00001-of-00002.safetensors: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4.97G/4.97G [01:58<00:00, 42.0MB/s] +model-00002-of-00002.safetensors: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1.46G/1.46G [00:34<00:00, 42.1MB/s] +Downloading shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [02:32<00:00, 76.49s/it] +Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:17<00:00, 8.55s/it] +generation_config.json: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 185/185 [00:00<00:00, 77.1kB/s] +Map: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 17519/17519 [00:47<00:00, 370.25 examples/s] +Map: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 18140/18140 [00:48<00:00, 371.24 examples/s] +tokenized_valid: Dataset({ + features: ['input_ids', 'attention_mask'], + num_rows: 1000 +}) +/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +[2024-11-13 18:07:23,603] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-11-13 18:07:33,759] [INFO] [comm.py:652:init_distributed] cdb=None +Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination +Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root... +Emitting ninja build file /home/chunhui/.cache/torch_extensions/py39_cu117/cpu_adam/build.ninja... +Building extension module cpu_adam... +Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +Loading extension module cpu_adam... +Time to load cpu_adam op: 11.725677490234375 seconds diff --git a/wandb/run-20241113_180230-zrr91ih2/files/wandb-metadata.json b/wandb/run-20241113_180230-zrr91ih2/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..bc610f5545fed4e0be2d6a58384819078cfc1a35 --- /dev/null +++ b/wandb/run-20241113_180230-zrr91ih2/files/wandb-metadata.json @@ -0,0 +1,97 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-11-13T23:02:30.039330Z", + "args": [ + "--perturbation", + "reverse_partial", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "3", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_ftp.py", + "codePath": "train/train_ftp.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_ftp.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA RTX A6000", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888559353856", + "used": "1745059610624" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + } + ], + "cudaVersion": "11.8" +} \ No newline at end of file diff --git a/wandb/run-20241113_180230-zrr91ih2/files/wandb-summary.json b/wandb/run-20241113_180230-zrr91ih2/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..7d8b6d062c357e9ddddd942ef1ba93de3bb13d25 --- /dev/null +++ b/wandb/run-20241113_180230-zrr91ih2/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":45879}} \ No newline at end of file diff --git a/wandb/run-20241113_180230-zrr91ih2/logs/debug-internal.log b/wandb/run-20241113_180230-zrr91ih2/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..c3798f6351ac391bfad51d066c1d1f4ea5ed9a60 --- /dev/null +++ b/wandb/run-20241113_180230-zrr91ih2/logs/debug-internal.log @@ -0,0 +1,17 @@ +{"time":"2024-11-13T18:02:30.042595611-05:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-11-13T18:02:30.042620041-05:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241113_180230-zrr91ih2/logs/debug-core.log"} +{"time":"2024-11-13T18:02:30.15316802-05:00","level":"INFO","msg":"created new stream","id":"zrr91ih2"} +{"time":"2024-11-13T18:02:30.15322945-05:00","level":"INFO","msg":"stream: started","id":"zrr91ih2"} +{"time":"2024-11-13T18:02:30.15332873-05:00","level":"INFO","msg":"sender: started","stream_id":"zrr91ih2"} +{"time":"2024-11-13T18:02:30.1532997-05:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"zrr91ih2"}} +{"time":"2024-11-13T18:02:30.153383301-05:00","level":"INFO","msg":"handler: started","stream_id":{"value":"zrr91ih2"}} +{"time":"2024-11-13T18:02:30.347287923-05:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-11-14T06:47:09.438255007-05:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2024-11-14T06:47:09.538459403-05:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2024-11-14T06:47:10.35272197-05:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"saving job artifact","runtime_seconds":0.646401489,"subtasks":[{"desc":"wandb-job.json","runtime_seconds":0.149693239,"progress":"556B/556B"}]}],"total_operations":1}} +{"time":"2024-11-14T06:47:11.366491743-05:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2024-11-14T06:47:12.481012736-05:00","level":"INFO","msg":"stream: closing","id":"zrr91ih2"} +{"time":"2024-11-14T06:47:12.481245138-05:00","level":"INFO","msg":"sender: closed","stream_id":"zrr91ih2"} +{"time":"2024-11-14T06:47:12.481046847-05:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"zrr91ih2"}} +{"time":"2024-11-14T06:47:12.481152027-05:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"zrr91ih2"}} +{"time":"2024-11-14T06:47:12.529070621-05:00","level":"INFO","msg":"stream: closed","id":"zrr91ih2"} diff --git a/wandb/run-20241113_180230-zrr91ih2/logs/debug.log b/wandb/run-20241113_180230-zrr91ih2/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..78ec5784c8413456ba00dd46955d273b41cf2e4c --- /dev/null +++ b/wandb/run-20241113_180230-zrr91ih2/logs/debug.log @@ -0,0 +1,33 @@ +2024-11-13 18:02:30,036 INFO MainThread:2553607 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-11-13 18:02:30,036 INFO MainThread:2553607 [wandb_setup.py:_flush():79] Configure stats pid to 2553607 +2024-11-13 18:02:30,036 INFO MainThread:2553607 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-11-13 18:02:30,036 INFO MainThread:2553607 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-11-13 18:02:30,036 INFO MainThread:2553607 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-13 18:02:30,036 INFO MainThread:2553607 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-13 18:02:30,036 INFO MainThread:2553607 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_ftp.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_ftp.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_ftp.py'} +2024-11-13 18:02:30,036 INFO MainThread:2553607 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-13 18:02:30,036 INFO MainThread:2553607 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241113_180230-zrr91ih2/logs/debug.log +2024-11-13 18:02:30,036 INFO MainThread:2553607 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241113_180230-zrr91ih2/logs/debug-internal.log +2024-11-13 18:02:30,036 INFO MainThread:2553607 [wandb_init.py:init():621] calling init triggers +2024-11-13 18:02:30,036 INFO MainThread:2553607 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-11-13 18:02:30,036 INFO MainThread:2553607 [wandb_init.py:init():671] starting backend +2024-11-13 18:02:30,037 INFO MainThread:2553607 [wandb_init.py:init():675] sending inform_init request +2024-11-13 18:02:30,038 INFO MainThread:2553607 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-13 18:02:30,039 INFO MainThread:2553607 [wandb_init.py:init():688] backend started and connected +2024-11-13 18:02:30,042 INFO MainThread:2553607 [wandb_init.py:init():783] updated telemetry +2024-11-13 18:02:30,065 INFO MainThread:2553607 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-11-13 18:02:30,344 INFO MainThread:2553607 [wandb_init.py:init():867] starting run threads in backend +2024-11-13 18:02:30,432 INFO MainThread:2553607 [wandb_run.py:_console_start():2463] atexit reg +2024-11-13 18:02:30,432 INFO MainThread:2553607 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-11-13 18:02:30,432 INFO MainThread:2553607 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-11-13 18:02:30,432 INFO MainThread:2553607 [wandb_run.py:_redirect():2401] Redirects installed. +2024-11-13 18:02:30,434 INFO MainThread:2553607 [wandb_init.py:init():911] run started, returning control to user process +2024-11-13 18:02:30,434 INFO MainThread:2553607 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_partial', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0, 'lr': 5e-06} +2024-11-14 06:47:09,190 INFO MainThread:2553607 [wandb_run.py:_finish():2158] finishing run yaning1001-dartmouth-college/exp-impo-reverse/zrr91ih2 +2024-11-14 06:47:09,255 INFO MainThread:2553607 [wandb_run.py:_atexit_cleanup():2426] got exitcode: 0 +2024-11-14 06:47:09,350 INFO MainThread:2553607 [wandb_run.py:_restore():2408] restore +2024-11-14 06:47:09,351 INFO MainThread:2553607 [wandb_run.py:_restore():2414] restore done +2024-11-14 06:47:12,469 INFO MainThread:2553607 [wandb_run.py:_footer_history_summary_info():3975] rendering history +2024-11-14 06:47:12,469 INFO MainThread:2553607 [wandb_run.py:_footer_history_summary_info():4007] rendering summary +2024-11-14 06:47:12,480 INFO MainThread:2553607 [wandb_run.py:_footer_sync_info():3934] logging synced files diff --git a/wandb/run-20241118_085831-3suh92jo/files/config.yaml b/wandb/run-20241118_085831-3suh92jo/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7c6db7f1d23f8df1cdf8299fcab2bac6152ba311 --- /dev/null +++ b/wandb/run-20241118_085831-3suh92jo/files/config.yaml @@ -0,0 +1,50 @@ +_wandb: + value: + cli_version: 0.18.5 + m: [] + python_version: 3.9.19 + t: + "1": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "2": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "3": + - 2 + - 13 + - 23 + - 55 + "4": 3.9.19 + "5": 0.18.5 + "6": 4.45.1 + "8": + - 5 + "12": 0.18.5 + "13": linux-x86_64 +batch_size: + value: 3 +epoch: + value: 3 +lr: + value: 5e-06 +perturbation: + value: shuffle_nondeterministic +seed: + value: 0 +train_set: + value: 10M diff --git a/wandb/run-20241118_085831-3suh92jo/files/output.log b/wandb/run-20241118_085831-3suh92jo/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..a835283386810bf2e9c0e686f045e45774392e03 --- /dev/null +++ b/wandb/run-20241118_085831-3suh92jo/files/output.log @@ -0,0 +1,29 @@ +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1098312/1098312 [00:03<00:00, 276585.44it/s] +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1098312/1098312 [00:00<00:00, 3218692.89it/s] +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 16426/16426 [00:00<00:00, 39902.42it/s] +Generating train split: 16425 examples [00:07, 2239.74 examples/s]█████████████████████████████████████▌ | 13490/16426 [00:00<00:00, 41691.09it/s] +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1085967/1085967 [00:04<00:00, 260631.76it/s] +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1085967/1085967 [00:00<00:00, 3588027.82it/s] +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 17014/17014 [00:00<00:00, 51154.27it/s] +Generating validation split: 17013 examples [00:07, 2247.40 examples/s]█████████████████████████████████████████████████████▋| 16974/17014 [00:00<00:00, 58795.17it/s] +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1031242/1031242 [00:03<00:00, 298493.66it/s] +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1031242/1031242 [00:00<00:00, 3642304.69it/s] +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 15439/15439 [00:00<00:00, 39952.83it/s] +Generating test split: 15438 examples [00:06, 2360.26 examples/s]████████████████████████████████████████████████████▎ | 14443/15439 [00:00<00:00, 44412.83it/s] +model.safetensors.index.json: 100%|██████████████████████████████████████████████████████████████████████████████████████████████| 20.9k/20.9k [00:00<00:00, 16.3MB/s] +Downloading shards: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [02:32<00:00, 76.24s/it] +Loading checkpoint shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:04<00:00, 2.17s/it] +Map: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████| 16425/16425 [00:45<00:00, 360.29 examples/s] +Map: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████| 17013/17013 [00:47<00:00, 359.03 examples/s] +tokenized_valid: Dataset({ + features: ['input_ids', 'attention_mask'], + num_rows: 1000 +}) +/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +[2024-11-18 09:03:05,599] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-11-18 09:03:12,413] [INFO] [comm.py:652:init_distributed] cdb=None +Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination +Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root... +Loading extension module cpu_adam... +Time to load cpu_adam op: 7.668454647064209 seconds diff --git a/wandb/run-20241118_085831-3suh92jo/files/wandb-metadata.json b/wandb/run-20241118_085831-3suh92jo/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..7358a3066b66af66eafd589024a30feb7fda21e7 --- /dev/null +++ b/wandb/run-20241118_085831-3suh92jo/files/wandb-metadata.json @@ -0,0 +1,97 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-11-18T13:58:31.935278Z", + "args": [ + "--perturbation", + "shuffle_nondeterministic", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "3", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_ftp.py", + "codePath": "train/train_ftp.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_ftp.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA RTX A6000", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888559353856", + "used": "1786847916032" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + } + ], + "cudaVersion": "11.8" +} \ No newline at end of file diff --git a/wandb/run-20241118_085831-3suh92jo/files/wandb-summary.json b/wandb/run-20241118_085831-3suh92jo/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..e8ae1757e62e760d5988c92a580e7d045310389f --- /dev/null +++ b/wandb/run-20241118_085831-3suh92jo/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":43009}} \ No newline at end of file diff --git a/wandb/run-20241118_085831-3suh92jo/logs/debug-internal.log b/wandb/run-20241118_085831-3suh92jo/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..555e306253fdf55a2c41936d970c76d1506a2f62 --- /dev/null +++ b/wandb/run-20241118_085831-3suh92jo/logs/debug-internal.log @@ -0,0 +1,17 @@ +{"time":"2024-11-18T08:58:31.938858201-05:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-11-18T08:58:31.938895162-05:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241118_085831-3suh92jo/logs/debug-core.log"} +{"time":"2024-11-18T08:58:32.053059977-05:00","level":"INFO","msg":"created new stream","id":"3suh92jo"} +{"time":"2024-11-18T08:58:32.053090937-05:00","level":"INFO","msg":"stream: started","id":"3suh92jo"} +{"time":"2024-11-18T08:58:32.053124237-05:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"3suh92jo"}} +{"time":"2024-11-18T08:58:32.053166867-05:00","level":"INFO","msg":"sender: started","stream_id":"3suh92jo"} +{"time":"2024-11-18T08:58:32.053142747-05:00","level":"INFO","msg":"handler: started","stream_id":{"value":"3suh92jo"}} +{"time":"2024-11-18T08:58:32.394778798-05:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-11-18T20:55:21.831088722-05:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2024-11-18T20:55:22.036125355-05:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2024-11-18T20:55:22.672315417-05:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2024-11-18T20:55:22.709572236-05:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2024-11-18T20:55:24.000683905-05:00","level":"INFO","msg":"stream: closing","id":"3suh92jo"} +{"time":"2024-11-18T20:55:24.000717545-05:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"3suh92jo"}} +{"time":"2024-11-18T20:55:24.000734105-05:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"3suh92jo"}} +{"time":"2024-11-18T20:55:24.000742895-05:00","level":"INFO","msg":"sender: closed","stream_id":"3suh92jo"} +{"time":"2024-11-18T20:55:24.000802396-05:00","level":"INFO","msg":"stream: closed","id":"3suh92jo"} diff --git a/wandb/run-20241118_085831-3suh92jo/logs/debug.log b/wandb/run-20241118_085831-3suh92jo/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..8867d90cc6b6eb9a167c06e82622f957eb313b22 --- /dev/null +++ b/wandb/run-20241118_085831-3suh92jo/logs/debug.log @@ -0,0 +1,33 @@ +2024-11-18 08:58:31,932 INFO MainThread:2672320 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-11-18 08:58:31,932 INFO MainThread:2672320 [wandb_setup.py:_flush():79] Configure stats pid to 2672320 +2024-11-18 08:58:31,932 INFO MainThread:2672320 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-11-18 08:58:31,932 INFO MainThread:2672320 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-11-18 08:58:31,933 INFO MainThread:2672320 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-18 08:58:31,933 INFO MainThread:2672320 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-18 08:58:31,933 INFO MainThread:2672320 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_ftp.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_ftp.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_ftp.py'} +2024-11-18 08:58:31,933 INFO MainThread:2672320 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-18 08:58:31,933 INFO MainThread:2672320 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241118_085831-3suh92jo/logs/debug.log +2024-11-18 08:58:31,933 INFO MainThread:2672320 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241118_085831-3suh92jo/logs/debug-internal.log +2024-11-18 08:58:31,933 INFO MainThread:2672320 [wandb_init.py:init():621] calling init triggers +2024-11-18 08:58:31,933 INFO MainThread:2672320 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-11-18 08:58:31,933 INFO MainThread:2672320 [wandb_init.py:init():671] starting backend +2024-11-18 08:58:31,933 INFO MainThread:2672320 [wandb_init.py:init():675] sending inform_init request +2024-11-18 08:58:31,934 INFO MainThread:2672320 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-18 08:58:31,935 INFO MainThread:2672320 [wandb_init.py:init():688] backend started and connected +2024-11-18 08:58:31,939 INFO MainThread:2672320 [wandb_init.py:init():783] updated telemetry +2024-11-18 08:58:31,967 INFO MainThread:2672320 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-11-18 08:58:32,391 INFO MainThread:2672320 [wandb_init.py:init():867] starting run threads in backend +2024-11-18 08:58:32,521 INFO MainThread:2672320 [wandb_run.py:_console_start():2463] atexit reg +2024-11-18 08:58:32,521 INFO MainThread:2672320 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-11-18 08:58:32,521 INFO MainThread:2672320 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-11-18 08:58:32,521 INFO MainThread:2672320 [wandb_run.py:_redirect():2401] Redirects installed. +2024-11-18 08:58:32,522 INFO MainThread:2672320 [wandb_init.py:init():911] run started, returning control to user process +2024-11-18 08:58:32,523 INFO MainThread:2672320 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_nondeterministic', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0, 'lr': 5e-06} +2024-11-18 20:55:21,528 INFO MainThread:2672320 [wandb_run.py:_finish():2158] finishing run yaning1001-dartmouth-college/exp-impo-shuffle/3suh92jo +2024-11-18 20:55:21,670 INFO MainThread:2672320 [wandb_run.py:_atexit_cleanup():2426] got exitcode: 0 +2024-11-18 20:55:21,671 INFO MainThread:2672320 [wandb_run.py:_restore():2408] restore +2024-11-18 20:55:21,671 INFO MainThread:2672320 [wandb_run.py:_restore():2414] restore done +2024-11-18 20:55:23,848 INFO MainThread:2672320 [wandb_run.py:_footer_history_summary_info():3975] rendering history +2024-11-18 20:55:23,848 INFO MainThread:2672320 [wandb_run.py:_footer_history_summary_info():4007] rendering summary +2024-11-18 20:55:23,961 INFO MainThread:2672320 [wandb_run.py:_footer_sync_info():3934] logging synced files