Add files using upload-large-folder tool

.gitattributes

@@ -39,3 +39,17 @@ wandb/run-20241101_012733-4u8e027p/run-4u8e027p.wandb filter=lfs diff=lfs merge=
 wandb/run-20241101_012733-e3zsr634/run-e3zsr634.wandb filter=lfs diff=lfs merge=lfs -text
 wandb/run-20241030_011509-3dp0dtmk/run-3dp0dtmk.wandb filter=lfs diff=lfs merge=lfs -text
 wandb/run-20241129_235322-dmnv987j/run-dmnv987j.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241105_161832-c18fx9uc/run-c18fx9uc.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241030_011509-46hc4g2h/run-46hc4g2h.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241129_235241-3ftwr60m/run-3ftwr60m.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241031_001055-u3ke9zn3/run-u3ke9zn3.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241113_180154-y8wvn5hq/run-y8wvn5hq.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241030_011509-oh4ul0xu/run-oh4ul0xu.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241030_225833-frh96rd1/run-frh96rd1.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241030_011509-a3dncwci/run-a3dncwci.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241030_013141-bkcoggdw/run-bkcoggdw.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241031_001055-32u9qnul/run-32u9qnul.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241030_231835-i7aqcsql/run-i7aqcsql.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241031_001055-sr4xke8e/run-sr4xke8e.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241031_114700-jx2hqvx3/run-jx2hqvx3.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241105_161832-sl992h9i/run-sl992h9i.wandb filter=lfs diff=lfs merge=lfs -text

wandb/run-20241030_011509-46hc4g2h/run-46hc4g2h.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1168507fc2efdaf39abd2e16fe247ea898936a7fd756db28b36941c1628320dd
+size 163840

wandb/run-20241030_011509-a3dncwci/run-a3dncwci.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31de4a80aa64e458b2e817e0bca2f90952a6a1bd55133b08076f86efb73888fe
+size 163840

wandb/run-20241030_011509-oh4ul0xu/run-oh4ul0xu.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5cae494d645173c2ab260b7431cfdd4cbe8a6d30ae2635b0edd4afb99e9a3102
+size 163840

wandb/run-20241030_013141-bkcoggdw/run-bkcoggdw.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ac456d5195118e26175a05980cc4f5d17ae96370198e1ba04d8c966b27252a0
+size 163840

wandb/run-20241030_225833-frh96rd1/run-frh96rd1.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70aaf1b9cc1adf99161f62fb97c95548b8cf7b0225e78cc55438924d0f2642cf
+size 1015808

wandb/run-20241030_231835-i7aqcsql/run-i7aqcsql.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c0bb6c061238472c5dba05ef529b177a7b24c4546adb8bd6c06a1dc34c306a75
+size 655360

wandb/run-20241030_233740-0fagh7s8/files/output.log

@@ -0,0 +1,190 @@
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:05<00:00,  2.60s/it]
+tokenized_valid: Dataset({
+    features: ['input_ids', 'attention_mask'],
+    num_rows: 600
+})
+/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
+  warnings.warn(
+[2024-10-30 23:37:50,273] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2024-10-30 23:37:58,862] [INFO] [comm.py:652:init_distributed] cdb=None
+[2024-10-30 23:37:58,863] [INFO] [comm.py:683:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl
+Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.
+Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination
+Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...
+Loading extension module cpu_adam...
+Time to load cpu_adam op: 4.667487144470215 seconds
+wandb: WARNING The `run_name` is currently set to the same value as `TrainingArguments.output_dir`. If this was not intended, please specify a different run name by setting the `TrainingArguments.run_name` parameter.
+                                                                                                                                                                                       
+{'loss': 2.7522, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.0}
+{'loss': 2.7541, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.0}
+{'loss': 2.8031, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.0}
+{'loss': 2.8062, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.0}
+{'loss': 2.7733, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.01}
+{'loss': 2.7949, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.01}
+{'loss': 2.7657, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.01}
+{'loss': 2.803, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.01}
+{'loss': 2.7793, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.01}
+{'loss': 2.7935, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.01}
+                                                                                                                                                                                       
+{'eval_loss': 2.812453508377075, 'eval_runtime': 12.4226, 'eval_samples_per_second': 48.299, 'eval_steps_per_second': 1.046, 'epoch': 0.01}
+{'loss': 2.7546, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.01}
+{'loss': 2.7423, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.01}
+{'loss': 2.8057, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.01}
+{'loss': 2.7663, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.01}
+{'loss': 2.7715, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.02}
+{'loss': 2.751, 'grad_norm': 4.9052934646606445, 'learning_rate': 4.998288843258043e-06, 'epoch': 0.02}
+{'loss': 2.6716, 'grad_norm': 4.931762218475342, 'learning_rate': 4.996577686516086e-06, 'epoch': 0.02}
+{'loss': 2.6561, 'grad_norm': 4.931762218475342, 'learning_rate': 4.996577686516086e-06, 'epoch': 0.02}
+{'loss': 2.579, 'grad_norm': 4.839107036590576, 'learning_rate': 4.994866529774127e-06, 'epoch': 0.02}
+{'loss': 2.5964, 'grad_norm': 3.662093162536621, 'learning_rate': 4.99315537303217e-06, 'epoch': 0.02}
+{'eval_loss': 2.5994045734405518, 'eval_runtime': 11.4634, 'eval_samples_per_second': 52.341, 'eval_steps_per_second': 1.134, 'epoch': 0.02}
+{'loss': 2.5669, 'grad_norm': 3.5343265533447266, 'learning_rate': 4.991444216290213e-06, 'epoch': 0.02}
+{'loss': 2.5163, 'grad_norm': 2.8458690643310547, 'learning_rate': 4.989733059548255e-06, 'epoch': 0.02}
+{'loss': 2.4443, 'grad_norm': 2.5915985107421875, 'learning_rate': 4.988021902806298e-06, 'epoch': 0.02}
+{'loss': 2.4325, 'grad_norm': 2.206981658935547, 'learning_rate': 4.9863107460643404e-06, 'epoch': 0.02}
+{'loss': 2.3948, 'grad_norm': 2.2773280143737793, 'learning_rate': 4.984599589322382e-06, 'epoch': 0.03}
+{'loss': 2.4063, 'grad_norm': 2.1317105293273926, 'learning_rate': 4.982888432580425e-06, 'epoch': 0.03}
+{'loss': 2.3657, 'grad_norm': 2.3999195098876953, 'learning_rate': 4.9811772758384674e-06, 'epoch': 0.03}
+{'loss': 2.3674, 'grad_norm': 1.9440910816192627, 'learning_rate': 4.97946611909651e-06, 'epoch': 0.03}
+{'loss': 2.3386, 'grad_norm': 2.343914747238159, 'learning_rate': 4.977754962354553e-06, 'epoch': 0.03}
+{'loss': 2.3157, 'grad_norm': 2.1667869091033936, 'learning_rate': 4.976043805612594e-06, 'epoch': 0.03}
+{'eval_loss': 2.3952019214630127, 'eval_runtime': 11.4922, 'eval_samples_per_second': 52.209, 'eval_steps_per_second': 1.131, 'epoch': 0.03}
+{'loss': 2.2711, 'grad_norm': 2.118894338607788, 'learning_rate': 4.974332648870637e-06, 'epoch': 0.03}
+{'loss': 2.3037, 'grad_norm': 2.242431640625, 'learning_rate': 4.972621492128679e-06, 'epoch': 0.03}
+{'loss': 2.2631, 'grad_norm': 1.9315046072006226, 'learning_rate': 4.970910335386721e-06, 'epoch': 0.03}
+{'loss': 2.2326, 'grad_norm': 2.2613368034362793, 'learning_rate': 4.969199178644764e-06, 'epoch': 0.03}
+{'loss': 2.2002, 'grad_norm': 1.979921579360962, 'learning_rate': 4.967488021902807e-06, 'epoch': 0.04}
+{'loss': 2.0944, 'grad_norm': 2.027212381362915, 'learning_rate': 4.965776865160849e-06, 'epoch': 0.04}
+{'loss': 2.2204, 'grad_norm': 2.199139356613159, 'learning_rate': 4.964065708418892e-06, 'epoch': 0.04}
+{'loss': 2.1221, 'grad_norm': 2.1344027519226074, 'learning_rate': 4.962354551676934e-06, 'epoch': 0.04}
+{'loss': 2.1284, 'grad_norm': 2.298335552215576, 'learning_rate': 4.960643394934976e-06, 'epoch': 0.04}
+{'loss': 2.1842, 'grad_norm': 2.3745176792144775, 'learning_rate': 4.958932238193019e-06, 'epoch': 0.04}
+{'eval_loss': 2.255570650100708, 'eval_runtime': 11.4984, 'eval_samples_per_second': 52.181, 'eval_steps_per_second': 1.131, 'epoch': 0.04}
+{'loss': 2.1337, 'grad_norm': 2.2507293224334717, 'learning_rate': 4.9572210814510614e-06, 'epoch': 0.04}
+{'loss': 2.0571, 'grad_norm': 2.3284757137298584, 'learning_rate': 4.955509924709104e-06, 'epoch': 0.04}
+{'loss': 2.1204, 'grad_norm': 2.172335624694824, 'learning_rate': 4.953798767967146e-06, 'epoch': 0.04}
+{'loss': 2.0751, 'grad_norm': 1.9553546905517578, 'learning_rate': 4.952087611225188e-06, 'epoch': 0.05}
+{'loss': 2.0911, 'grad_norm': 2.0383987426757812, 'learning_rate': 4.950376454483231e-06, 'epoch': 0.05}
+{'loss': 2.06, 'grad_norm': 1.9470635652542114, 'learning_rate': 4.948665297741274e-06, 'epoch': 0.05}
+{'loss': 2.1145, 'grad_norm': 1.7316076755523682, 'learning_rate': 4.946954140999316e-06, 'epoch': 0.05}
+{'loss': 2.0498, 'grad_norm': 1.9633406400680542, 'learning_rate': 4.945242984257359e-06, 'epoch': 0.05}
+{'loss': 2.0609, 'grad_norm': 1.8334592580795288, 'learning_rate': 4.943531827515401e-06, 'epoch': 0.05}
+{'loss': 1.988, 'grad_norm': 1.8270950317382812, 'learning_rate': 4.941820670773443e-06, 'epoch': 0.05}
+{'eval_loss': 2.1854093074798584, 'eval_runtime': 11.5011, 'eval_samples_per_second': 52.169, 'eval_steps_per_second': 1.13, 'epoch': 0.05}
+{'loss': 1.989, 'grad_norm': 1.8450721502304077, 'learning_rate': 4.940109514031486e-06, 'epoch': 0.05}
+{'loss': 2.0996, 'grad_norm': 2.2471470832824707, 'learning_rate': 4.9383983572895284e-06, 'epoch': 0.05}
+{'loss': 1.9723, 'grad_norm': 1.864260196685791, 'learning_rate': 4.936687200547571e-06, 'epoch': 0.05}
+{'loss': 2.0376, 'grad_norm': 2.2607100009918213, 'learning_rate': 4.934976043805613e-06, 'epoch': 0.06}
+{'loss': 1.9901, 'grad_norm': 1.9184648990631104, 'learning_rate': 4.9332648870636554e-06, 'epoch': 0.06}
+{'loss': 1.9581, 'grad_norm': 1.981023907661438, 'learning_rate': 4.931553730321697e-06, 'epoch': 0.06}
+{'loss': 2.0003, 'grad_norm': 2.0862128734588623, 'learning_rate': 4.92984257357974e-06, 'epoch': 0.06}
+{'loss': 1.9247, 'grad_norm': 1.8063560724258423, 'learning_rate': 4.928131416837782e-06, 'epoch': 0.06}
+{'loss': 1.9957, 'grad_norm': 1.9832228422164917, 'learning_rate': 4.926420260095825e-06, 'epoch': 0.06}
+{'loss': 1.9882, 'grad_norm': 2.1227223873138428, 'learning_rate': 4.924709103353868e-06, 'epoch': 0.06}
+{'eval_loss': 2.138824462890625, 'eval_runtime': 11.5113, 'eval_samples_per_second': 52.123, 'eval_steps_per_second': 1.129, 'epoch': 0.06}
+{'loss': 1.9439, 'grad_norm': 2.0159411430358887, 'learning_rate': 4.92299794661191e-06, 'epoch': 0.06}
+{'loss': 1.9647, 'grad_norm': 2.2820119857788086, 'learning_rate': 4.921286789869952e-06, 'epoch': 0.06}
+{'loss': 2.012, 'grad_norm': 2.0620205402374268, 'learning_rate': 4.919575633127995e-06, 'epoch': 0.06}
+{'loss': 1.9966, 'grad_norm': 2.2670867443084717, 'learning_rate': 4.917864476386037e-06, 'epoch': 0.07}
+{'loss': 1.8535, 'grad_norm': 2.0552380084991455, 'learning_rate': 4.91615331964408e-06, 'epoch': 0.07}
+{'loss': 1.9662, 'grad_norm': 2.675915241241455, 'learning_rate': 4.9144421629021224e-06, 'epoch': 0.07}
+{'loss': 1.9008, 'grad_norm': 2.1373021602630615, 'learning_rate': 4.912731006160164e-06, 'epoch': 0.07}
+{'loss': 1.979, 'grad_norm': 2.100844621658325, 'learning_rate': 4.911019849418207e-06, 'epoch': 0.07}
+{'loss': 1.9173, 'grad_norm': 2.0223195552825928, 'learning_rate': 4.9093086926762494e-06, 'epoch': 0.07}
+{'loss': 2.0353, 'grad_norm': 2.165973663330078, 'learning_rate': 4.907597535934292e-06, 'epoch': 0.07}
+{'eval_loss': 2.1140758991241455, 'eval_runtime': 11.5235, 'eval_samples_per_second': 52.068, 'eval_steps_per_second': 1.128, 'epoch': 0.07}
+{'loss': 1.9297, 'grad_norm': 2.223146677017212, 'learning_rate': 4.905886379192335e-06, 'epoch': 0.07}
+{'loss': 1.8733, 'grad_norm': 2.062178373336792, 'learning_rate': 4.904175222450377e-06, 'epoch': 0.07}
+{'loss': 1.9402, 'grad_norm': 2.034820318222046, 'learning_rate': 4.902464065708419e-06, 'epoch': 0.07}
+{'loss': 1.9037, 'grad_norm': 2.516028881072998, 'learning_rate': 4.900752908966462e-06, 'epoch': 0.08}
+{'loss': 1.9109, 'grad_norm': 2.4920814037323, 'learning_rate': 4.899041752224504e-06, 'epoch': 0.08}
+{'loss': 1.9042, 'grad_norm': 2.6503076553344727, 'learning_rate': 4.897330595482547e-06, 'epoch': 0.08}
+{'loss': 1.9213, 'grad_norm': 2.3713159561157227, 'learning_rate': 4.8956194387405895e-06, 'epoch': 0.08}
+{'loss': 1.9263, 'grad_norm': 2.7794573307037354, 'learning_rate': 4.893908281998632e-06, 'epoch': 0.08}
+{'loss': 1.8679, 'grad_norm': 2.1413917541503906, 'learning_rate': 4.892197125256674e-06, 'epoch': 0.08}
+{'loss': 1.9298, 'grad_norm': 2.5015697479248047, 'learning_rate': 4.8904859685147164e-06, 'epoch': 0.08}
+{'eval_loss': 2.0753817558288574, 'eval_runtime': 11.5583, 'eval_samples_per_second': 51.911, 'eval_steps_per_second': 1.125, 'epoch': 0.08}
+{'loss': 1.8715, 'grad_norm': 2.1490447521209717, 'learning_rate': 4.888774811772759e-06, 'epoch': 0.08}
+{'loss': 1.9024, 'grad_norm': 2.6189815998077393, 'learning_rate': 4.887063655030802e-06, 'epoch': 0.08}
+{'loss': 1.8966, 'grad_norm': 2.212202310562134, 'learning_rate': 4.8853524982888434e-06, 'epoch': 0.09}
+{'loss': 1.8856, 'grad_norm': 2.5672545433044434, 'learning_rate': 4.883641341546886e-06, 'epoch': 0.09}
+{'loss': 1.854, 'grad_norm': 2.3435440063476562, 'learning_rate': 4.881930184804929e-06, 'epoch': 0.09}
+{'loss': 1.9035, 'grad_norm': 2.4554519653320312, 'learning_rate': 4.88021902806297e-06, 'epoch': 0.09}
+{'loss': 1.8662, 'grad_norm': 2.377504825592041, 'learning_rate': 4.878507871321013e-06, 'epoch': 0.09}
+{'loss': 1.8593, 'grad_norm': 2.3757731914520264, 'learning_rate': 4.876796714579056e-06, 'epoch': 0.09}
+{'loss': 1.8779, 'grad_norm': 2.5893397331237793, 'learning_rate': 4.875085557837098e-06, 'epoch': 0.09}
+{'loss': 1.8233, 'grad_norm': 2.3579938411712646, 'learning_rate': 4.873374401095141e-06, 'epoch': 0.09}
+{'eval_loss': 2.0544168949127197, 'eval_runtime': 11.5311, 'eval_samples_per_second': 52.033, 'eval_steps_per_second': 1.127, 'epoch': 0.09}
+{'loss': 1.8527, 'grad_norm': 2.5492780208587646, 'learning_rate': 4.8716632443531835e-06, 'epoch': 0.09}
+{'loss': 1.8841, 'grad_norm': 2.0388078689575195, 'learning_rate': 4.869952087611225e-06, 'epoch': 0.09}
+{'loss': 1.872, 'grad_norm': 2.2521777153015137, 'learning_rate': 4.868240930869268e-06, 'epoch': 0.1}
+{'loss': 1.8546, 'grad_norm': 2.197253704071045, 'learning_rate': 4.8665297741273105e-06, 'epoch': 0.1}
+{'loss': 1.9132, 'grad_norm': 2.158266544342041, 'learning_rate': 4.864818617385353e-06, 'epoch': 0.1}
+{'loss': 1.7997, 'grad_norm': 2.40034556388855, 'learning_rate': 4.863107460643396e-06, 'epoch': 0.1}
+{'loss': 1.874, 'grad_norm': 2.6923694610595703, 'learning_rate': 4.8613963039014374e-06, 'epoch': 0.1}
+{'loss': 1.7818, 'grad_norm': 2.381399631500244, 'learning_rate': 4.85968514715948e-06, 'epoch': 0.1}
+{'loss': 1.7879, 'grad_norm': 2.354761838912964, 'learning_rate': 4.857973990417523e-06, 'epoch': 0.1}
+{'loss': 1.8158, 'grad_norm': 2.516366720199585, 'learning_rate': 4.856262833675565e-06, 'epoch': 0.1}
+{'eval_loss': 2.036221981048584, 'eval_runtime': 11.5436, 'eval_samples_per_second': 51.977, 'eval_steps_per_second': 1.126, 'epoch': 0.1}
+{'loss': 1.83, 'grad_norm': 2.0844292640686035, 'learning_rate': 4.854551676933608e-06, 'epoch': 0.1}
+{'loss': 1.8088, 'grad_norm': 2.376721143722534, 'learning_rate': 4.8528405201916505e-06, 'epoch': 0.1}
+{'loss': 1.7779, 'grad_norm': 2.107591390609741, 'learning_rate': 4.851129363449692e-06, 'epoch': 0.11}
+{'loss': 1.7956, 'grad_norm': 2.4341471195220947, 'learning_rate': 4.849418206707735e-06, 'epoch': 0.11}
+{'loss': 1.8709, 'grad_norm': 2.6235601902008057, 'learning_rate': 4.8477070499657775e-06, 'epoch': 0.11}
+{'loss': 1.8434, 'grad_norm': 2.254868984222412, 'learning_rate': 4.84599589322382e-06, 'epoch': 0.11}
+{'loss': 1.8517, 'grad_norm': 2.4880223274230957, 'learning_rate': 4.844284736481863e-06, 'epoch': 0.11}
+{'loss': 1.8268, 'grad_norm': 2.358351945877075, 'learning_rate': 4.8425735797399045e-06, 'epoch': 0.11}
+{'loss': 1.7414, 'grad_norm': 2.4404406547546387, 'learning_rate': 4.840862422997947e-06, 'epoch': 0.11}
+{'loss': 1.7884, 'grad_norm': 2.3423702716827393, 'learning_rate': 4.839151266255989e-06, 'epoch': 0.11}
+{'eval_loss': 2.0196220874786377, 'eval_runtime': 11.5245, 'eval_samples_per_second': 52.063, 'eval_steps_per_second': 1.128, 'epoch': 0.11}
+{'loss': 1.7282, 'grad_norm': 2.115130662918091, 'learning_rate': 4.8374401095140314e-06, 'epoch': 0.11}
+{'loss': 1.8166, 'grad_norm': 2.571716547012329, 'learning_rate': 4.835728952772074e-06, 'epoch': 0.11}
+{'loss': 1.7803, 'grad_norm': 2.3601763248443604, 'learning_rate': 4.834017796030117e-06, 'epoch': 0.12}
+{'loss': 1.7986, 'grad_norm': 2.492929220199585, 'learning_rate': 4.832306639288159e-06, 'epoch': 0.12}
+{'loss': 1.7451, 'grad_norm': 2.342839479446411, 'learning_rate': 4.830595482546202e-06, 'epoch': 0.12}
+{'loss': 1.8559, 'grad_norm': 2.339629650115967, 'learning_rate': 4.828884325804244e-06, 'epoch': 0.12}
+{'loss': 1.788, 'grad_norm': 2.605527639389038, 'learning_rate': 4.827173169062286e-06, 'epoch': 0.12}
+{'loss': 1.8042, 'grad_norm': 2.326620578765869, 'learning_rate': 4.825462012320329e-06, 'epoch': 0.12}
+{'loss': 1.8078, 'grad_norm': 2.5194342136383057, 'learning_rate': 4.8237508555783715e-06, 'epoch': 0.12}
+{'loss': 1.8827, 'grad_norm': 2.5342624187469482, 'learning_rate': 4.822039698836414e-06, 'epoch': 0.12}
+{'eval_loss': 2.013911247253418, 'eval_runtime': 11.5397, 'eval_samples_per_second': 51.994, 'eval_steps_per_second': 1.127, 'epoch': 0.12}
+{'loss': 1.8128, 'grad_norm': 2.2788140773773193, 'learning_rate': 4.820328542094456e-06, 'epoch': 0.12}
+{'loss': 1.7875, 'grad_norm': 2.805044651031494, 'learning_rate': 4.8186173853524985e-06, 'epoch': 0.13}
+{'loss': 1.7445, 'grad_norm': 2.242846727371216, 'learning_rate': 4.816906228610541e-06, 'epoch': 0.13}
+{'loss': 1.7738, 'grad_norm': 3.2011518478393555, 'learning_rate': 4.815195071868584e-06, 'epoch': 0.13}
+{'loss': 1.7282, 'grad_norm': 2.282356023788452, 'learning_rate': 4.813483915126626e-06, 'epoch': 0.13}
+{'loss': 1.7981, 'grad_norm': 2.6586058139801025, 'learning_rate': 4.811772758384669e-06, 'epoch': 0.13}
+{'loss': 1.7205, 'grad_norm': 2.262554407119751, 'learning_rate': 4.810061601642711e-06, 'epoch': 0.13}
+{'loss': 1.7682, 'grad_norm': 2.705533266067505, 'learning_rate': 4.808350444900753e-06, 'epoch': 0.13}
+{'loss': 1.8276, 'grad_norm': 2.4971351623535156, 'learning_rate': 4.806639288158796e-06, 'epoch': 0.13}
+{'loss': 1.8191, 'grad_norm': 2.391136884689331, 'learning_rate': 4.8049281314168385e-06, 'epoch': 0.13}
+{'eval_loss': 1.9947330951690674, 'eval_runtime': 11.5227, 'eval_samples_per_second': 52.071, 'eval_steps_per_second': 1.128, 'epoch': 0.13}
+{'loss': 1.7417, 'grad_norm': 2.3191349506378174, 'learning_rate': 4.803216974674881e-06, 'epoch': 0.13}
+{'loss': 1.8142, 'grad_norm': 2.5745604038238525, 'learning_rate': 4.801505817932923e-06, 'epoch': 0.14}
+{'loss': 1.7814, 'grad_norm': 2.19454026222229, 'learning_rate': 4.7997946611909655e-06, 'epoch': 0.14}
+{'loss': 1.7395, 'grad_norm': 2.351425886154175, 'learning_rate': 4.798083504449008e-06, 'epoch': 0.14}
+{'loss': 1.7758, 'grad_norm': 2.460624933242798, 'learning_rate': 4.796372347707051e-06, 'epoch': 0.14}
+{'loss': 1.7903, 'grad_norm': 2.3103744983673096, 'learning_rate': 4.7946611909650925e-06, 'epoch': 0.14}
+{'loss': 1.7159, 'grad_norm': 2.410243272781372, 'learning_rate': 4.792950034223135e-06, 'epoch': 0.14}
+{'loss': 1.7891, 'grad_norm': 2.3782598972320557, 'learning_rate': 4.791238877481178e-06, 'epoch': 0.14}
+{'loss': 1.7334, 'grad_norm': 2.193119525909424, 'learning_rate': 4.78952772073922e-06, 'epoch': 0.14}
+{'loss': 1.7392, 'grad_norm': 2.43776535987854, 'learning_rate': 4.787816563997262e-06, 'epoch': 0.14}
+{'eval_loss': 1.9786564111709595, 'eval_runtime': 11.5696, 'eval_samples_per_second': 51.86, 'eval_steps_per_second': 1.124, 'epoch': 0.14}
+{'loss': 1.7707, 'grad_norm': 2.404174566268921, 'learning_rate': 4.786105407255305e-06, 'epoch': 0.14}
+{'loss': 1.7237, 'grad_norm': 2.301892042160034, 'learning_rate': 4.784394250513347e-06, 'epoch': 0.15}
+{'loss': 1.8541, 'grad_norm': 2.5396673679351807, 'learning_rate': 4.78268309377139e-06, 'epoch': 0.15}
+{'loss': 1.7333, 'grad_norm': 2.522949457168579, 'learning_rate': 4.7809719370294325e-06, 'epoch': 0.15}
+{'loss': 1.7192, 'grad_norm': 2.1934566497802734, 'learning_rate': 4.779260780287474e-06, 'epoch': 0.15}
+{'loss': 1.7228, 'grad_norm': 2.566537618637085, 'learning_rate': 4.777549623545517e-06, 'epoch': 0.15}
+{'loss': 1.7478, 'grad_norm': 2.3536360263824463, 'learning_rate': 4.7758384668035595e-06, 'epoch': 0.15}
+{'loss': 1.7562, 'grad_norm': 2.2443368434906006, 'learning_rate': 4.774127310061602e-06, 'epoch': 0.15}
+{'loss': 1.841, 'grad_norm': 2.413248300552368, 'learning_rate': 4.772416153319645e-06, 'epoch': 0.15}
+{'loss': 1.7732, 'grad_norm': 2.375981330871582, 'learning_rate': 4.770704996577687e-06, 'epoch': 0.15}
+{'eval_loss': 1.982629418373108, 'eval_runtime': 11.522, 'eval_samples_per_second': 52.074, 'eval_steps_per_second': 1.128, 'epoch': 0.15}
+{'loss': 1.6788, 'grad_norm': 2.263878107070923, 'learning_rate': 4.768993839835729e-06, 'epoch': 0.16}
+{'loss': 1.7174, 'grad_norm': 2.600919723510742, 'learning_rate': 4.767282683093772e-06, 'epoch': 0.16}
+{'loss': 1.7688, 'grad_norm': 2.5808844566345215, 'learning_rate': 4.765571526351814e-06, 'epoch': 0.16}
+{'loss': 1.7607, 'grad_norm': 2.5976321697235107, 'learning_rate': 4.763860369609857e-06, 'epoch': 0.16}
+{'loss': 1.7058, 'grad_norm': 2.2662572860717773, 'learning_rate': 4.7621492128678995e-06, 'epoch': 0.16}
+{'loss': 1.682, 'grad_norm': 2.3581748008728027, 'learning_rate': 4.760438056125941e-06, 'epoch': 0.16}
+{'loss': 1.739, 'grad_norm': 2.254718780517578, 'learning_rate': 4.758726899383984e-06, 'epoch': 0.16}

wandb/run-20241030_233740-0fagh7s8/files/requirements.txt

@@ -0,0 +1,147 @@
+funcsigs==1.0.2
+sentry-sdk==2.17.0
+multiprocess==0.70.16
+numpy==1.26.2
+pluralizer==1.2.0
+debugpy==1.6.7
+nvidia-cudnn-cu11==8.5.0.96
+deepspeed==0.15.2
+data==0.4
+pandas==2.1.3
+tomli==2.0.1
+charset-normalizer==3.3.2
+attrs==24.2.0
+aiosignal==1.3.1
+fsspec==2023.10.0
+nvidia-cusparse-cu11==11.7.4.91
+zipp==3.12.0
+mypy-extensions==1.0.0
+datasets==3.0.1
+joblib==1.3.2
+hjson==3.1.0
+traitlets==5.7.1
+stack-data==0.6.0
+transformers==4.45.1
+sympy==1.11.1
+Pygments==2.15.0
+docker-pycreds==0.4.0
+dill==0.3.8
+wheel==0.44.0
+prompt-toolkit==3.0.30
+parso==0.8.3
+ipykernel==6.23.1
+pyarrow==17.0.0
+certifi==2023.11.17
+nvidia-cufft-cu11==10.9.0.58
+six==1.16.0
+pydantic==2.9.2
+click==8.1.7
+nest-asyncio==1.5.6
+gmpy2==2.1.0
+matplotlib==3.8.2
+scipy==1.11.4
+typing_extensions==4.12.2
+statsmodels==0.14.0
+huggingface-hub==0.25.0
+frozenlist==1.4.1
+gpustat==1.1.1
+nvidia-nvtx-cu11==11.7.91
+safetensors==0.4.5
+stanza==1.9.2
+decorator==5.1.1
+seaborn==0.13.0
+sentencepiece==0.2.0
+PyYAML==6.0.1
+black==24.8.0
+protobuf==4.25.1
+pickleshare==0.7.5
+peft==0.13.0
+triton==2.0.0
+nvidia-cuda-runtime-cu11==11.7.99
+Jinja2==3.1.2
+nvidia-cusolver-cu11==11.4.0.1
+executing==1.2.0
+jupyter_client==8.1.0
+pluggy==1.3.0
+cmake==3.30.3
+pytz==2023.3.post1
+aiohappyeyeballs==2.4.2
+kiwisolver==1.4.5
+py-cpuinfo==9.0.0
+Pillow==10.1.0
+ptyprocess==0.7.0
+importlib_resources==6.4.5
+GitPython==3.1.43
+importlib-metadata==6.0.0
+iniconfig==2.0.0
+scikit-learn==1.3.2
+exceptiongroup==1.1.0
+networkx==2.8.6
+accelerate==1.0.0
+nltk==3.8.1
+shutilwhich==1.1.0
+fonttools==4.45.1
+future==0.18.3
+aiohttp==3.10.6
+wcwidth==0.2.5
+idna==3.6
+filelock==3.12.2
+pathspec==0.12.1
+jupyter_core==5.1.0
+lit==18.1.8
+nvidia-curand-cu11==10.2.10.91
+nvidia-cublas-cu11==11.10.3.66
+nvidia-ml-py==12.560.30
+msgpack==1.1.0
+python-dateutil==2.8.2
+blessed==1.20.0
+packaging==23.0
+gitdb==4.0.11
+yarl==1.13.0
+emoji==2.8.0
+tzdata==2023.3
+cycler==0.12.1
+tornado==6.2
+backcall==0.2.0
+plotnine==0.12.4
+ninja==1.11.1.1
+latex==0.7.0
+wandb==0.18.5
+setproctitle==1.3.3
+threadpoolctl==3.2.0
+requests==2.32.3
+pyparsing==3.1.1
+smmap==5.0.1
+pyzmq==23.0.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+matplotlib-inline==0.1.6
+latexcodec==1.0.0
+ipython==8.0.0
+patsy==0.5.3
+contourpy==1.2.0
+multidict==6.1.0
+mizani==0.9.3
+urllib3==2.1.0
+tokenizers==0.20.0
+MarkupSafe==2.1.2
+pip==24.2
+pexpect==4.8.0
+tqdm==4.66.5
+jedi==0.18.2
+pydantic_core==2.23.4
+tempdir==0.7.1
+mpmath==1.2.1
+setuptools==72.1.0
+pytest==7.4.3
+pure-eval==0.2.2
+psutil==5.9.1
+comm==0.1.2
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+regex==2023.10.3
+platformdirs==2.5.2
+asttokens==2.2.1
+torch==2.0.0
+nvidia-nccl-cu11==2.14.3
+xxhash==3.5.0

wandb/run-20241030_233740-0fagh7s8/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-10-31T03:37:40.842788Z",
+  "args":  [
+    "--perturbation",
+    "reverse_control",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "3",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1711065919488"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241030_233740-0fagh7s8/logs/debug-internal.log

@@ -0,0 +1,8 @@
+{"time":"2024-10-30T23:37:40.847295918-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-10-30T23:37:40.847310969-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_233740-0fagh7s8/logs/debug-core.log"}
+{"time":"2024-10-30T23:37:40.9597517-04:00","level":"INFO","msg":"created new stream","id":"0fagh7s8"}
+{"time":"2024-10-30T23:37:40.95982373-04:00","level":"INFO","msg":"stream: started","id":"0fagh7s8"}
+{"time":"2024-10-30T23:37:40.95985659-04:00","level":"INFO","msg":"sender: started","stream_id":"0fagh7s8"}
+{"time":"2024-10-30T23:37:40.959899061-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"0fagh7s8"}}
+{"time":"2024-10-30T23:37:40.95984633-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"0fagh7s8"}}
+{"time":"2024-10-30T23:37:41.417240986-04:00","level":"INFO","msg":"Starting system monitor"}

wandb/run-20241030_233740-0fagh7s8/logs/debug.log

@@ -0,0 +1,29 @@
+2024-10-30 23:37:40,840 INFO    MainThread:464533 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-10-30 23:37:40,840 INFO    MainThread:464533 [wandb_setup.py:_flush():79] Configure stats pid to 464533
+2024-10-30 23:37:40,840 INFO    MainThread:464533 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-10-30 23:37:40,840 INFO    MainThread:464533 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-10-30 23:37:40,840 INFO    MainThread:464533 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-10-30 23:37:40,840 INFO    MainThread:464533 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-30 23:37:40,840 INFO    MainThread:464533 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-10-30 23:37:40,840 INFO    MainThread:464533 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-30 23:37:40,841 INFO    MainThread:464533 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_233740-0fagh7s8/logs/debug.log
+2024-10-30 23:37:40,841 INFO    MainThread:464533 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_233740-0fagh7s8/logs/debug-internal.log
+2024-10-30 23:37:40,841 INFO    MainThread:464533 [wandb_init.py:init():621] calling init triggers
+2024-10-30 23:37:40,841 INFO    MainThread:464533 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-10-30 23:37:40,841 INFO    MainThread:464533 [wandb_init.py:init():671] starting backend
+2024-10-30 23:37:40,841 INFO    MainThread:464533 [wandb_init.py:init():675] sending inform_init request
+2024-10-30 23:37:40,842 INFO    MainThread:464533 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-30 23:37:40,842 INFO    MainThread:464533 [wandb_init.py:init():688] backend started and connected
+2024-10-30 23:37:40,845 INFO    MainThread:464533 [wandb_init.py:init():783] updated telemetry
+2024-10-30 23:37:40,872 INFO    MainThread:464533 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-10-30 23:37:41,413 INFO    MainThread:464533 [wandb_init.py:init():867] starting run threads in backend
+2024-10-30 23:37:42,836 INFO    MainThread:464533 [wandb_run.py:_console_start():2463] atexit reg
+2024-10-30 23:37:42,836 INFO    MainThread:464533 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-10-30 23:37:42,836 INFO    MainThread:464533 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-10-30 23:37:42,837 INFO    MainThread:464533 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-10-30 23:37:42,860 INFO    MainThread:464533 [wandb_init.py:init():911] run started, returning control to user process
+2024-10-30 23:37:42,861 INFO    MainThread:464533 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_control', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0}
+2024-10-30 23:38:26,525 INFO    MainThread:464533 [wandb_run.py:_config_callback():1390] config_cb None None {'vocab_size': 128256, 'max_position_embeddings': 131072, 'hidden_size': 3072, 'intermediate_size': 8192, 'num_hidden_layers': 28, 'num_attention_heads': 24, 'num_key_value_heads': 8, 'hidden_act': 'silu', 'initializer_range': 0.02, 'rms_norm_eps': 1e-05, 'pretraining_tp': 1, 'use_cache': True, 'rope_theta': 500000.0, 'rope_scaling': {'factor': 32.0, 'high_freq_factor': 4.0, 'low_freq_factor': 1.0, 'original_max_position_embeddings': 8192, 'rope_type': 'llama3'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 128, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'bfloat16', 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': True, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['LlamaForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 128000, 'pad_token_id': None, 'eos_token_id': 128001, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'meta-llama/Llama-3.2-3B', 'transformers_version': '4.45.1', 'model_type': 'llama', 'output_dir': './checkpoints/Llama-3.2-3B/babylm_reverse_control_10M_seed0/runs', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': True, 'do_predict': False, 'eval_strategy': 'steps', 'prediction_loss_only': False, 'per_device_train_batch_size': 3, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'eval_delay': 0, 'torch_empty_cache_steps': None, 'learning_rate': 5e-06, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 3, 'max_steps': -1, 'lr_scheduler_type': 'linear', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.0, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': './logs', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 1, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 150, 'save_total_limit': None, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 0, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': False, 'fp16': True, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 10, 'dataloader_num_workers': 0, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': './checkpoints/Llama-3.2-3B/babylm_reverse_control_10M_seed0/runs', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': 'deepspeed_config/train_dp_config.json', 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'evaluation_strategy': 'steps', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False, 'use_liger_kernel': False, 'eval_use_gather_object': False}
+2024-10-30 23:38:26,527 INFO    MainThread:464533 [wandb_config.py:__setitem__():154] config set model/num_parameters = 3212749824 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x7fef9f509f40>>
+2024-10-30 23:38:26,527 INFO    MainThread:464533 [wandb_run.py:_config_callback():1390] config_cb model/num_parameters 3212749824 None

wandb/run-20241031_001055-32u9qnul/run-32u9qnul.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f49988f8aae635634599adabf52569773e38cf954305e4c2a3d38c8e8c46c83
+size 327680

wandb/run-20241031_001055-sr4xke8e/run-sr4xke8e.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:155772d9839e7a89851871c99ac7de107a6aff85919294aa49fe9a729aa05b42
+size 327680

wandb/run-20241031_001055-u3ke9zn3/run-u3ke9zn3.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2e42dc976478e26e3507c297f038b31d00be976a0b3e1b6f4bd2c1bfc0dbbef
+size 327680

wandb/run-20241031_114700-jx2hqvx3/run-jx2hqvx3.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48b33ce646e8a0f568b819405b479cdfeccc2daae28acd906189efa464154fd6
+size 1015808

wandb/run-20241031_122113-yzubvl65/files/config.yaml

@@ -0,0 +1,50 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.9.19
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "3":
+                - 2
+                - 13
+                - 23
+                - 55
+            "4": 3.9.19
+            "5": 0.18.5
+            "6": 4.45.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+batch_size:
+    value: 3
+epoch:
+    value: 6
+lr:
+    value: 5e-06
+perturbation:
+    value: reverse_full
+seed:
+    value: 0
+train_set:
+    value: 10M

wandb/run-20241031_122113-yzubvl65/files/output.log

@@ -0,0 +1,15 @@
+model-00001-of-00002.safetensors: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████| 4.97G/4.97G [01:34<00:00, 42.2MB/s]
+Downloading shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [02:09<00:00, 64.69s/it]
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:05<00:00,  2.57s/it]
+tokenized_valid: Dataset({
+    features: ['input_ids', 'attention_mask'],
+    num_rows: 600
+})
+/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
+  warnings.warn(
+[2024-10-31 12:23:30,557] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2024-10-31 12:23:39,160] [INFO] [comm.py:652:init_distributed] cdb=None
+Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination
+Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...
+Loading extension module cpu_adam...
+Time to load cpu_adam op: 5.0282368659973145 seconds

wandb/run-20241031_122113-yzubvl65/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-10-31T16:21:13.915797Z",
+  "args":  [
+    "--perturbation",
+    "reverse_full",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "6",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1753159962624"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241031_122113-yzubvl65/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":32015}}

wandb/run-20241101_092804-l47v8jb3/files/output.log

@@ -0,0 +1,17 @@
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:14<00:00,  7.41s/it]
+Map: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 17013/17013 [00:47<00:00, 360.48 examples/s]
+tokenized_valid: Dataset({
+    features: ['input_ids', 'attention_mask'],
+    num_rows: 600
+})
+/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
+  warnings.warn(
+[2024-11-01 09:29:08,573] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2024-11-01 09:29:18,017] [INFO] [comm.py:652:init_distributed] cdb=None
+Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination
+Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...
+Emitting ninja build file /home/chunhui/.cache/torch_extensions/py39_cu117/cpu_adam/build.ninja...
+Building extension module cpu_adam...
+Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N)
+Loading extension module cpu_adam...
+Time to load cpu_adam op: 5.2360241413116455 seconds

wandb/run-20241101_092804-l47v8jb3/files/requirements.txt

@@ -0,0 +1,147 @@
+funcsigs==1.0.2
+sentry-sdk==2.17.0
+multiprocess==0.70.16
+numpy==1.26.2
+pluralizer==1.2.0
+debugpy==1.6.7
+nvidia-cudnn-cu11==8.5.0.96
+deepspeed==0.15.2
+data==0.4
+pandas==2.1.3
+tomli==2.0.1
+charset-normalizer==3.3.2
+attrs==24.2.0
+aiosignal==1.3.1
+fsspec==2023.10.0
+nvidia-cusparse-cu11==11.7.4.91
+zipp==3.12.0
+mypy-extensions==1.0.0
+datasets==3.0.1
+joblib==1.3.2
+hjson==3.1.0
+traitlets==5.7.1
+stack-data==0.6.0
+transformers==4.45.1
+sympy==1.11.1
+Pygments==2.15.0
+docker-pycreds==0.4.0
+dill==0.3.8
+wheel==0.44.0
+prompt-toolkit==3.0.30
+parso==0.8.3
+ipykernel==6.23.1
+pyarrow==17.0.0
+certifi==2023.11.17
+nvidia-cufft-cu11==10.9.0.58
+six==1.16.0
+pydantic==2.9.2
+click==8.1.7
+nest-asyncio==1.5.6
+gmpy2==2.1.0
+matplotlib==3.8.2
+scipy==1.11.4
+typing_extensions==4.12.2
+statsmodels==0.14.0
+huggingface-hub==0.25.0
+frozenlist==1.4.1
+gpustat==1.1.1
+nvidia-nvtx-cu11==11.7.91
+safetensors==0.4.5
+stanza==1.9.2
+decorator==5.1.1
+seaborn==0.13.0
+sentencepiece==0.2.0
+PyYAML==6.0.1
+black==24.8.0
+protobuf==4.25.1
+pickleshare==0.7.5
+peft==0.13.0
+triton==2.0.0
+nvidia-cuda-runtime-cu11==11.7.99
+Jinja2==3.1.2
+nvidia-cusolver-cu11==11.4.0.1
+executing==1.2.0
+jupyter_client==8.1.0
+pluggy==1.3.0
+cmake==3.30.3
+pytz==2023.3.post1
+aiohappyeyeballs==2.4.2
+kiwisolver==1.4.5
+py-cpuinfo==9.0.0
+Pillow==10.1.0
+ptyprocess==0.7.0
+importlib_resources==6.4.5
+GitPython==3.1.43
+importlib-metadata==6.0.0
+iniconfig==2.0.0
+scikit-learn==1.3.2
+exceptiongroup==1.1.0
+networkx==2.8.6
+accelerate==1.0.0
+nltk==3.8.1
+shutilwhich==1.1.0
+fonttools==4.45.1
+future==0.18.3
+aiohttp==3.10.6
+wcwidth==0.2.5
+idna==3.6
+filelock==3.12.2
+pathspec==0.12.1
+jupyter_core==5.1.0
+lit==18.1.8
+nvidia-curand-cu11==10.2.10.91
+nvidia-cublas-cu11==11.10.3.66
+nvidia-ml-py==12.560.30
+msgpack==1.1.0
+python-dateutil==2.8.2
+blessed==1.20.0
+packaging==23.0
+gitdb==4.0.11
+yarl==1.13.0
+emoji==2.8.0
+tzdata==2023.3
+cycler==0.12.1
+tornado==6.2
+backcall==0.2.0
+plotnine==0.12.4
+ninja==1.11.1.1
+latex==0.7.0
+wandb==0.18.5
+setproctitle==1.3.3
+threadpoolctl==3.2.0
+requests==2.32.3
+pyparsing==3.1.1
+smmap==5.0.1
+pyzmq==23.0.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+matplotlib-inline==0.1.6
+latexcodec==1.0.0
+ipython==8.0.0
+patsy==0.5.3
+contourpy==1.2.0
+multidict==6.1.0
+mizani==0.9.3
+urllib3==2.1.0
+tokenizers==0.20.0
+MarkupSafe==2.1.2
+pip==24.2
+pexpect==4.8.0
+tqdm==4.66.5
+jedi==0.18.2
+pydantic_core==2.23.4
+tempdir==0.7.1
+mpmath==1.2.1
+setuptools==72.1.0
+pytest==7.4.3
+pure-eval==0.2.2
+psutil==5.9.1
+comm==0.1.2
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+regex==2023.10.3
+platformdirs==2.5.2
+asttokens==2.2.1
+torch==2.0.0
+nvidia-nccl-cu11==2.14.3
+xxhash==3.5.0

wandb/run-20241101_092804-l47v8jb3/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-11-01T13:28:04.307193Z",
+  "args":  [
+    "--perturbation",
+    "shuffle_nondeterministic",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "6",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1754628071424"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241101_092804-l47v8jb3/logs/debug-internal.log

@@ -0,0 +1,8 @@
+{"time":"2024-11-01T09:28:04.310194036-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-11-01T09:28:04.310206806-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_092804-l47v8jb3/logs/debug-core.log"}
+{"time":"2024-11-01T09:28:04.417666825-04:00","level":"INFO","msg":"created new stream","id":"l47v8jb3"}
+{"time":"2024-11-01T09:28:04.417704745-04:00","level":"INFO","msg":"stream: started","id":"l47v8jb3"}
+{"time":"2024-11-01T09:28:04.417757526-04:00","level":"INFO","msg":"sender: started","stream_id":"l47v8jb3"}
+{"time":"2024-11-01T09:28:04.417744946-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"l47v8jb3"}}
+{"time":"2024-11-01T09:28:04.417757796-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"l47v8jb3"}}
+{"time":"2024-11-01T09:28:04.604158521-04:00","level":"INFO","msg":"Starting system monitor"}

wandb/run-20241101_092804-l47v8jb3/logs/debug.log

@@ -0,0 +1,26 @@
+2024-11-01 09:28:04,304 INFO    MainThread:779818 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-11-01 09:28:04,304 INFO    MainThread:779818 [wandb_setup.py:_flush():79] Configure stats pid to 779818
+2024-11-01 09:28:04,304 INFO    MainThread:779818 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-11-01 09:28:04,304 INFO    MainThread:779818 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-11-01 09:28:04,305 INFO    MainThread:779818 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-11-01 09:28:04,305 INFO    MainThread:779818 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-11-01 09:28:04,305 INFO    MainThread:779818 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-11-01 09:28:04,305 INFO    MainThread:779818 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-11-01 09:28:04,305 INFO    MainThread:779818 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_092804-l47v8jb3/logs/debug.log
+2024-11-01 09:28:04,305 INFO    MainThread:779818 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_092804-l47v8jb3/logs/debug-internal.log
+2024-11-01 09:28:04,305 INFO    MainThread:779818 [wandb_init.py:init():621] calling init triggers
+2024-11-01 09:28:04,305 INFO    MainThread:779818 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-11-01 09:28:04,305 INFO    MainThread:779818 [wandb_init.py:init():671] starting backend
+2024-11-01 09:28:04,305 INFO    MainThread:779818 [wandb_init.py:init():675] sending inform_init request
+2024-11-01 09:28:04,306 INFO    MainThread:779818 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-11-01 09:28:04,307 INFO    MainThread:779818 [wandb_init.py:init():688] backend started and connected
+2024-11-01 09:28:04,311 INFO    MainThread:779818 [wandb_init.py:init():783] updated telemetry
+2024-11-01 09:28:04,339 INFO    MainThread:779818 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-11-01 09:28:04,601 INFO    MainThread:779818 [wandb_init.py:init():867] starting run threads in backend
+2024-11-01 09:28:04,698 INFO    MainThread:779818 [wandb_run.py:_console_start():2463] atexit reg
+2024-11-01 09:28:04,698 INFO    MainThread:779818 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-11-01 09:28:04,698 INFO    MainThread:779818 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-11-01 09:28:04,698 INFO    MainThread:779818 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-11-01 09:28:04,700 INFO    MainThread:779818 [wandb_init.py:init():911] run started, returning control to user process
+2024-11-01 09:28:04,700 INFO    MainThread:779818 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_nondeterministic', 'train_set': '10M', 'batch_size': 3, 'epoch': 6, 'seed': 0, 'lr': 5e-06}

wandb/run-20241101_200502-0isq1nei/files/output.log

@@ -0,0 +1 @@
+Loading checkpoint shards:   0%|                                                                                                                                                        | 0/2 [00:00<?, ?it/s]

wandb/run-20241101_200502-0isq1nei/files/requirements.txt

@@ -0,0 +1,147 @@
+funcsigs==1.0.2
+sentry-sdk==2.17.0
+multiprocess==0.70.16
+numpy==1.26.2
+pluralizer==1.2.0
+debugpy==1.6.7
+nvidia-cudnn-cu11==8.5.0.96
+deepspeed==0.15.2
+data==0.4
+pandas==2.1.3
+tomli==2.0.1
+charset-normalizer==3.3.2
+attrs==24.2.0
+aiosignal==1.3.1
+fsspec==2023.10.0
+nvidia-cusparse-cu11==11.7.4.91
+zipp==3.12.0
+mypy-extensions==1.0.0
+datasets==3.0.1
+joblib==1.3.2
+hjson==3.1.0
+traitlets==5.7.1
+stack-data==0.6.0
+transformers==4.45.1
+sympy==1.11.1
+Pygments==2.15.0
+docker-pycreds==0.4.0
+dill==0.3.8
+wheel==0.44.0
+prompt-toolkit==3.0.30
+parso==0.8.3
+ipykernel==6.23.1
+pyarrow==17.0.0
+certifi==2023.11.17
+nvidia-cufft-cu11==10.9.0.58
+six==1.16.0
+pydantic==2.9.2
+click==8.1.7
+nest-asyncio==1.5.6
+gmpy2==2.1.0
+matplotlib==3.8.2
+scipy==1.11.4
+typing_extensions==4.12.2
+statsmodels==0.14.0
+huggingface-hub==0.25.0
+frozenlist==1.4.1
+gpustat==1.1.1
+nvidia-nvtx-cu11==11.7.91
+safetensors==0.4.5
+stanza==1.9.2
+decorator==5.1.1
+seaborn==0.13.0
+sentencepiece==0.2.0
+PyYAML==6.0.1
+black==24.8.0
+protobuf==4.25.1
+pickleshare==0.7.5
+peft==0.13.0
+triton==2.0.0
+nvidia-cuda-runtime-cu11==11.7.99
+Jinja2==3.1.2
+nvidia-cusolver-cu11==11.4.0.1
+executing==1.2.0
+jupyter_client==8.1.0
+pluggy==1.3.0
+cmake==3.30.3
+pytz==2023.3.post1
+aiohappyeyeballs==2.4.2
+kiwisolver==1.4.5
+py-cpuinfo==9.0.0
+Pillow==10.1.0
+ptyprocess==0.7.0
+importlib_resources==6.4.5
+GitPython==3.1.43
+importlib-metadata==6.0.0
+iniconfig==2.0.0
+scikit-learn==1.3.2
+exceptiongroup==1.1.0
+networkx==2.8.6
+accelerate==1.0.0
+nltk==3.8.1
+shutilwhich==1.1.0
+fonttools==4.45.1
+future==0.18.3
+aiohttp==3.10.6
+wcwidth==0.2.5
+idna==3.6
+filelock==3.12.2
+pathspec==0.12.1
+jupyter_core==5.1.0
+lit==18.1.8
+nvidia-curand-cu11==10.2.10.91
+nvidia-cublas-cu11==11.10.3.66
+nvidia-ml-py==12.560.30
+msgpack==1.1.0
+python-dateutil==2.8.2
+blessed==1.20.0
+packaging==23.0
+gitdb==4.0.11
+yarl==1.13.0
+emoji==2.8.0
+tzdata==2023.3
+cycler==0.12.1
+tornado==6.2
+backcall==0.2.0
+plotnine==0.12.4
+ninja==1.11.1.1
+latex==0.7.0
+wandb==0.18.5
+setproctitle==1.3.3
+threadpoolctl==3.2.0
+requests==2.32.3
+pyparsing==3.1.1
+smmap==5.0.1
+pyzmq==23.0.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+matplotlib-inline==0.1.6
+latexcodec==1.0.0
+ipython==8.0.0
+patsy==0.5.3
+contourpy==1.2.0
+multidict==6.1.0
+mizani==0.9.3
+urllib3==2.1.0
+tokenizers==0.20.0
+MarkupSafe==2.1.2
+pip==24.2
+pexpect==4.8.0
+tqdm==4.66.5
+jedi==0.18.2
+pydantic_core==2.23.4
+tempdir==0.7.1
+mpmath==1.2.1
+setuptools==72.1.0
+pytest==7.4.3
+pure-eval==0.2.2
+psutil==5.9.1
+comm==0.1.2
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+regex==2023.10.3
+platformdirs==2.5.2
+asttokens==2.2.1
+torch==2.0.0
+nvidia-nccl-cu11==2.14.3
+xxhash==3.5.0

wandb/run-20241101_200502-0isq1nei/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-11-02T00:05:02.630386Z",
+  "args":  [
+    "--perturbation",
+    "shuffle_nondeterministic",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "3",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1754801463296"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241101_200502-0isq1nei/logs/debug-internal.log

@@ -0,0 +1,8 @@
+{"time":"2024-11-01T20:05:02.634312951-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-11-01T20:05:02.634334801-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_200502-0isq1nei/logs/debug-core.log"}
+{"time":"2024-11-01T20:05:02.845016287-04:00","level":"INFO","msg":"created new stream","id":"0isq1nei"}
+{"time":"2024-11-01T20:05:02.845066148-04:00","level":"INFO","msg":"stream: started","id":"0isq1nei"}
+{"time":"2024-11-01T20:05:02.845110768-04:00","level":"INFO","msg":"sender: started","stream_id":"0isq1nei"}
+{"time":"2024-11-01T20:05:02.845080408-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"0isq1nei"}}
+{"time":"2024-11-01T20:05:02.845109518-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"0isq1nei"}}
+{"time":"2024-11-01T20:05:03.078764068-04:00","level":"INFO","msg":"Starting system monitor"}

wandb/run-20241101_200502-0isq1nei/logs/debug.log

@@ -0,0 +1,26 @@
+2024-11-01 20:05:02,626 INFO    MainThread:869514 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-11-01 20:05:02,626 INFO    MainThread:869514 [wandb_setup.py:_flush():79] Configure stats pid to 869514
+2024-11-01 20:05:02,626 INFO    MainThread:869514 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-11-01 20:05:02,627 INFO    MainThread:869514 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-11-01 20:05:02,627 INFO    MainThread:869514 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-11-01 20:05:02,627 INFO    MainThread:869514 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-11-01 20:05:02,627 INFO    MainThread:869514 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-11-01 20:05:02,627 INFO    MainThread:869514 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-11-01 20:05:02,627 INFO    MainThread:869514 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_200502-0isq1nei/logs/debug.log
+2024-11-01 20:05:02,627 INFO    MainThread:869514 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_200502-0isq1nei/logs/debug-internal.log
+2024-11-01 20:05:02,627 INFO    MainThread:869514 [wandb_init.py:init():621] calling init triggers
+2024-11-01 20:05:02,627 INFO    MainThread:869514 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-11-01 20:05:02,627 INFO    MainThread:869514 [wandb_init.py:init():671] starting backend
+2024-11-01 20:05:02,627 INFO    MainThread:869514 [wandb_init.py:init():675] sending inform_init request
+2024-11-01 20:05:02,629 INFO    MainThread:869514 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-11-01 20:05:02,630 INFO    MainThread:869514 [wandb_init.py:init():688] backend started and connected
+2024-11-01 20:05:02,634 INFO    MainThread:869514 [wandb_init.py:init():783] updated telemetry
+2024-11-01 20:05:02,655 INFO    MainThread:869514 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-11-01 20:05:03,076 INFO    MainThread:869514 [wandb_init.py:init():867] starting run threads in backend
+2024-11-01 20:05:03,164 INFO    MainThread:869514 [wandb_run.py:_console_start():2463] atexit reg
+2024-11-01 20:05:03,164 INFO    MainThread:869514 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-11-01 20:05:03,164 INFO    MainThread:869514 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-11-01 20:05:03,164 INFO    MainThread:869514 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-11-01 20:05:03,166 INFO    MainThread:869514 [wandb_init.py:init():911] run started, returning control to user process
+2024-11-01 20:05:03,166 INFO    MainThread:869514 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_nondeterministic', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0, 'lr': 5e-06}

wandb/run-20241101_201630-85vocaw4/files/config.yaml

@@ -0,0 +1,49 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.9.19
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "3":
+                - 13
+                - 23
+                - 55
+            "4": 3.9.19
+            "5": 0.18.5
+            "6": 4.45.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+batch_size:
+    value: 3
+epoch:
+    value: 6
+lr:
+    value: 5e-06
+perturbation:
+    value: shuffle_nodeterministic
+seed:
+    value: 0
+train_set:
+    value: 10M

wandb/run-20241101_201630-85vocaw4/files/output.log

@@ -0,0 +1,12 @@
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 164, in <module>
+    dataset = load_dataset('babylm_dataset_test.py', name=dataset_name, trust_remote_code=True)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/load.py", line 2074, in load_dataset
+    builder_instance = load_dataset_builder(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/load.py", line 1832, in load_dataset_builder
+    builder_instance: DatasetBuilder = builder_cls(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/builder.py", line 342, in __init__
+    self.config, self.config_id = self._create_builder_config(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/builder.py", line 569, in _create_builder_config
+    raise ValueError(
+ValueError: BuilderConfig 'babylm_shuffle_nodeterministic_10M_seed0' not found. Available: ['babylm_hop_control_10M_seed0', 'babylm_hop_tokens4_10M_seed0', 'babylm_hop_words4_10M_seed0', 'babylm_reverse_control_10M_seed0', 'babylm_reverse_partial_10M_seed0', 'babylm_reverse_full_10M_seed0', 'babylm_shuffle_control_10M_seed0', 'babylm_shuffle_nondeterministic_10M_seed0', 'babylm_shuffle_deterministic21_10M_seed0', 'babylm_shuffle_deterministic57_10M_seed0', 'babylm_shuffle_deterministic84_10M_seed0', 'babylm_shuffle_local3_10M_seed0', 'babylm_shuffle_local5_10M_seed0', 'babylm_shuffle_local10_10M_seed0', 'babylm_shuffle_even_odd_10M_seed0']

wandb/run-20241101_201630-85vocaw4/files/wandb-metadata.json

@@ -0,0 +1,29 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-11-02T00:16:30.588147Z",
+  "args":  [
+    "--perturbation",
+    "shuffle_nodeterministic",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "6",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py"
+}

wandb/run-20241101_201630-85vocaw4/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":0}}

wandb/run-20241101_201630-85vocaw4/logs/debug-internal.log

@@ -0,0 +1,16 @@
+{"time":"2024-11-01T20:16:30.590081773-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-11-01T20:16:30.590096543-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_201630-85vocaw4/logs/debug-core.log"}
+{"time":"2024-11-01T20:16:30.698509196-04:00","level":"INFO","msg":"created new stream","id":"85vocaw4"}
+{"time":"2024-11-01T20:16:30.698571246-04:00","level":"INFO","msg":"stream: started","id":"85vocaw4"}
+{"time":"2024-11-01T20:16:30.698659457-04:00","level":"INFO","msg":"sender: started","stream_id":"85vocaw4"}
+{"time":"2024-11-01T20:16:30.698637567-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"85vocaw4"}}
+{"time":"2024-11-01T20:16:30.698633617-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"85vocaw4"}}
+{"time":"2024-11-01T20:16:30.872065521-04:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-11-01T20:16:30.969121491-04:00","level":"INFO","msg":"stream: closing","id":"85vocaw4"}
+{"time":"2024-11-01T20:16:30.969183031-04:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-11-01T20:16:30.977041665-04:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-11-01T20:16:31.508663323-04:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-11-01T20:16:31.620535336-04:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"85vocaw4"}}
+{"time":"2024-11-01T20:16:31.620624207-04:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"85vocaw4"}}
+{"time":"2024-11-01T20:16:31.620662317-04:00","level":"INFO","msg":"sender: closed","stream_id":"85vocaw4"}
+{"time":"2024-11-01T20:16:31.620703757-04:00","level":"INFO","msg":"stream: closed","id":"85vocaw4"}

wandb/run-20241101_201630-85vocaw4/logs/debug.log

@@ -0,0 +1,27 @@
+2024-11-01 20:16:30,586 INFO    MainThread:874719 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-11-01 20:16:30,586 INFO    MainThread:874719 [wandb_setup.py:_flush():79] Configure stats pid to 874719
+2024-11-01 20:16:30,586 INFO    MainThread:874719 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-11-01 20:16:30,586 INFO    MainThread:874719 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-11-01 20:16:30,586 INFO    MainThread:874719 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-11-01 20:16:30,586 INFO    MainThread:874719 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-11-01 20:16:30,586 INFO    MainThread:874719 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-11-01 20:16:30,586 INFO    MainThread:874719 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-11-01 20:16:30,586 INFO    MainThread:874719 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_201630-85vocaw4/logs/debug.log
+2024-11-01 20:16:30,586 INFO    MainThread:874719 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_201630-85vocaw4/logs/debug-internal.log
+2024-11-01 20:16:30,586 INFO    MainThread:874719 [wandb_init.py:init():621] calling init triggers
+2024-11-01 20:16:30,586 INFO    MainThread:874719 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-11-01 20:16:30,586 INFO    MainThread:874719 [wandb_init.py:init():671] starting backend
+2024-11-01 20:16:30,586 INFO    MainThread:874719 [wandb_init.py:init():675] sending inform_init request
+2024-11-01 20:16:30,587 INFO    MainThread:874719 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-11-01 20:16:30,588 INFO    MainThread:874719 [wandb_init.py:init():688] backend started and connected
+2024-11-01 20:16:30,591 INFO    MainThread:874719 [wandb_init.py:init():783] updated telemetry
+2024-11-01 20:16:30,625 INFO    MainThread:874719 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-11-01 20:16:30,868 INFO    MainThread:874719 [wandb_init.py:init():867] starting run threads in backend
+2024-11-01 20:16:30,958 INFO    MainThread:874719 [wandb_run.py:_console_start():2463] atexit reg
+2024-11-01 20:16:30,958 INFO    MainThread:874719 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-11-01 20:16:30,958 INFO    MainThread:874719 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-11-01 20:16:30,958 INFO    MainThread:874719 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-11-01 20:16:30,960 INFO    MainThread:874719 [wandb_init.py:init():911] run started, returning control to user process
+2024-11-01 20:16:30,961 INFO    MainThread:874719 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_nodeterministic', 'train_set': '10M', 'batch_size': 3, 'epoch': 6, 'seed': 0, 'lr': 5e-06}
+2024-11-01 20:16:30,969 WARNING MsgRouterThr:874719 [router.py:message_loop():77] message_loop has been closed

wandb/run-20241101_201630-cb0m59vi/files/config.yaml

@@ -0,0 +1,49 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.9.19
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "3":
+                - 13
+                - 23
+                - 55
+            "4": 3.9.19
+            "5": 0.18.5
+            "6": 4.45.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+batch_size:
+    value: 3
+epoch:
+    value: 6
+lr:
+    value: 5e-06
+perturbation:
+    value: shuffle_nodeterministic
+seed:
+    value: 0
+train_set:
+    value: 10M

wandb/run-20241101_201630-cb0m59vi/files/output.log

@@ -0,0 +1,12 @@
+Traceback (most recent call last):
+  File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 164, in <module>
+    dataset = load_dataset('babylm_dataset_test.py', name=dataset_name, trust_remote_code=True)
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/load.py", line 2074, in load_dataset
+    builder_instance = load_dataset_builder(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/load.py", line 1832, in load_dataset_builder
+    builder_instance: DatasetBuilder = builder_cls(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/builder.py", line 342, in __init__
+    self.config, self.config_id = self._create_builder_config(
+  File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/builder.py", line 569, in _create_builder_config
+    raise ValueError(
+ValueError: BuilderConfig 'babylm_shuffle_nodeterministic_10M_seed0' not found. Available: ['babylm_hop_control_10M_seed0', 'babylm_hop_tokens4_10M_seed0', 'babylm_hop_words4_10M_seed0', 'babylm_reverse_control_10M_seed0', 'babylm_reverse_partial_10M_seed0', 'babylm_reverse_full_10M_seed0', 'babylm_shuffle_control_10M_seed0', 'babylm_shuffle_nondeterministic_10M_seed0', 'babylm_shuffle_deterministic21_10M_seed0', 'babylm_shuffle_deterministic57_10M_seed0', 'babylm_shuffle_deterministic84_10M_seed0', 'babylm_shuffle_local3_10M_seed0', 'babylm_shuffle_local5_10M_seed0', 'babylm_shuffle_local10_10M_seed0', 'babylm_shuffle_even_odd_10M_seed0']

wandb/run-20241101_201630-cb0m59vi/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-11-02T00:16:30.561525Z",
+  "args":  [
+    "--perturbation",
+    "shuffle_nodeterministic",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "6",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1754802601984"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241101_201630-cb0m59vi/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":0}}

wandb/run-20241101_201630-cb0m59vi/logs/debug-internal.log

@@ -0,0 +1,16 @@
+{"time":"2024-11-01T20:16:30.563400835-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-11-01T20:16:30.563412425-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_201630-cb0m59vi/logs/debug-core.log"}
+{"time":"2024-11-01T20:16:30.671139114-04:00","level":"INFO","msg":"created new stream","id":"cb0m59vi"}
+{"time":"2024-11-01T20:16:30.671176224-04:00","level":"INFO","msg":"stream: started","id":"cb0m59vi"}
+{"time":"2024-11-01T20:16:30.671275055-04:00","level":"INFO","msg":"sender: started","stream_id":"cb0m59vi"}
+{"time":"2024-11-01T20:16:30.671219504-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"cb0m59vi"}}
+{"time":"2024-11-01T20:16:30.671207154-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"cb0m59vi"}}
+{"time":"2024-11-01T20:16:30.839933803-04:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-11-01T20:16:30.936897352-04:00","level":"INFO","msg":"stream: closing","id":"cb0m59vi"}
+{"time":"2024-11-01T20:16:30.936931672-04:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-11-01T20:16:30.937377334-04:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-11-01T20:16:31.485218533-04:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-11-01T20:16:31.608476399-04:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"cb0m59vi"}}
+{"time":"2024-11-01T20:16:31.608528239-04:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"cb0m59vi"}}
+{"time":"2024-11-01T20:16:31.608545179-04:00","level":"INFO","msg":"sender: closed","stream_id":"cb0m59vi"}
+{"time":"2024-11-01T20:16:31.608580439-04:00","level":"INFO","msg":"stream: closed","id":"cb0m59vi"}

wandb/run-20241101_201630-cb0m59vi/logs/debug.log

@@ -0,0 +1,27 @@
+2024-11-01 20:16:30,559 INFO    MainThread:874714 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-11-01 20:16:30,559 INFO    MainThread:874714 [wandb_setup.py:_flush():79] Configure stats pid to 874714
+2024-11-01 20:16:30,559 INFO    MainThread:874714 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings
+2024-11-01 20:16:30,559 INFO    MainThread:874714 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings
+2024-11-01 20:16:30,559 INFO    MainThread:874714 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2024-11-01 20:16:30,559 INFO    MainThread:874714 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-11-01 20:16:30,559 INFO    MainThread:874714 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'}
+2024-11-01 20:16:30,559 INFO    MainThread:874714 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-11-01 20:16:30,559 INFO    MainThread:874714 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_201630-cb0m59vi/logs/debug.log
+2024-11-01 20:16:30,559 INFO    MainThread:874714 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_201630-cb0m59vi/logs/debug-internal.log
+2024-11-01 20:16:30,559 INFO    MainThread:874714 [wandb_init.py:init():621] calling init triggers
+2024-11-01 20:16:30,559 INFO    MainThread:874714 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {}
+2024-11-01 20:16:30,559 INFO    MainThread:874714 [wandb_init.py:init():671] starting backend
+2024-11-01 20:16:30,559 INFO    MainThread:874714 [wandb_init.py:init():675] sending inform_init request
+2024-11-01 20:16:30,560 INFO    MainThread:874714 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-11-01 20:16:30,561 INFO    MainThread:874714 [wandb_init.py:init():688] backend started and connected
+2024-11-01 20:16:30,565 INFO    MainThread:874714 [wandb_init.py:init():783] updated telemetry
+2024-11-01 20:16:30,586 INFO    MainThread:874714 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-11-01 20:16:30,837 INFO    MainThread:874714 [wandb_init.py:init():867] starting run threads in backend
+2024-11-01 20:16:30,926 INFO    MainThread:874714 [wandb_run.py:_console_start():2463] atexit reg
+2024-11-01 20:16:30,926 INFO    MainThread:874714 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-11-01 20:16:30,926 INFO    MainThread:874714 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-11-01 20:16:30,926 INFO    MainThread:874714 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-11-01 20:16:30,927 INFO    MainThread:874714 [wandb_init.py:init():911] run started, returning control to user process
+2024-11-01 20:16:30,927 INFO    MainThread:874714 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_nodeterministic', 'train_set': '10M', 'batch_size': 3, 'epoch': 6, 'seed': 0, 'lr': 5e-06}
+2024-11-01 20:16:30,937 WARNING MsgRouterThr:874714 [router.py:message_loop():77] message_loop has been closed

wandb/run-20241101_201707-vk4mc2rm/files/output.log

@@ -0,0 +1,17 @@
+Loading checkpoint shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:04<00:00,  2.49s/it]
+tokenized_valid: Dataset({
+    features: ['input_ids', 'attention_mask'],
+    num_rows: 600
+})
+/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
+  warnings.warn(
+[2024-11-01 20:17:16,784] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2024-11-01 20:17:25,921] [INFO] [comm.py:652:init_distributed] cdb=None
+[2024-11-01 20:17:25,921] [INFO] [comm.py:683:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl
+Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.
+Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination
+Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...
+Loading extension module cpu_adam...
+Time to load cpu_adam op: 5.009636640548706 seconds
+wandb: WARNING The `run_name` is currently set to the same value as `TrainingArguments.output_dir`. If this was not intended, please specify a different run name by setting the `TrainingArguments.run_name` parameter.
+  0%|                                                                                                                                                                                | 0/2739 [00:00<?, ?it/s]

wandb/run-20241101_201707-vk4mc2rm/files/requirements.txt

@@ -0,0 +1,147 @@
+funcsigs==1.0.2
+sentry-sdk==2.17.0
+multiprocess==0.70.16
+numpy==1.26.2
+pluralizer==1.2.0
+debugpy==1.6.7
+nvidia-cudnn-cu11==8.5.0.96
+deepspeed==0.15.2
+data==0.4
+pandas==2.1.3
+tomli==2.0.1
+charset-normalizer==3.3.2
+attrs==24.2.0
+aiosignal==1.3.1
+fsspec==2023.10.0
+nvidia-cusparse-cu11==11.7.4.91
+zipp==3.12.0
+mypy-extensions==1.0.0
+datasets==3.0.1
+joblib==1.3.2
+hjson==3.1.0
+traitlets==5.7.1
+stack-data==0.6.0
+transformers==4.45.1
+sympy==1.11.1
+Pygments==2.15.0
+docker-pycreds==0.4.0
+dill==0.3.8
+wheel==0.44.0
+prompt-toolkit==3.0.30
+parso==0.8.3
+ipykernel==6.23.1
+pyarrow==17.0.0
+certifi==2023.11.17
+nvidia-cufft-cu11==10.9.0.58
+six==1.16.0
+pydantic==2.9.2
+click==8.1.7
+nest-asyncio==1.5.6
+gmpy2==2.1.0
+matplotlib==3.8.2
+scipy==1.11.4
+typing_extensions==4.12.2
+statsmodels==0.14.0
+huggingface-hub==0.25.0
+frozenlist==1.4.1
+gpustat==1.1.1
+nvidia-nvtx-cu11==11.7.91
+safetensors==0.4.5
+stanza==1.9.2
+decorator==5.1.1
+seaborn==0.13.0
+sentencepiece==0.2.0
+PyYAML==6.0.1
+black==24.8.0
+protobuf==4.25.1
+pickleshare==0.7.5
+peft==0.13.0
+triton==2.0.0
+nvidia-cuda-runtime-cu11==11.7.99
+Jinja2==3.1.2
+nvidia-cusolver-cu11==11.4.0.1
+executing==1.2.0
+jupyter_client==8.1.0
+pluggy==1.3.0
+cmake==3.30.3
+pytz==2023.3.post1
+aiohappyeyeballs==2.4.2
+kiwisolver==1.4.5
+py-cpuinfo==9.0.0
+Pillow==10.1.0
+ptyprocess==0.7.0
+importlib_resources==6.4.5
+GitPython==3.1.43
+importlib-metadata==6.0.0
+iniconfig==2.0.0
+scikit-learn==1.3.2
+exceptiongroup==1.1.0
+networkx==2.8.6
+accelerate==1.0.0
+nltk==3.8.1
+shutilwhich==1.1.0
+fonttools==4.45.1
+future==0.18.3
+aiohttp==3.10.6
+wcwidth==0.2.5
+idna==3.6
+filelock==3.12.2
+pathspec==0.12.1
+jupyter_core==5.1.0
+lit==18.1.8
+nvidia-curand-cu11==10.2.10.91
+nvidia-cublas-cu11==11.10.3.66
+nvidia-ml-py==12.560.30
+msgpack==1.1.0
+python-dateutil==2.8.2
+blessed==1.20.0
+packaging==23.0
+gitdb==4.0.11
+yarl==1.13.0
+emoji==2.8.0
+tzdata==2023.3
+cycler==0.12.1
+tornado==6.2
+backcall==0.2.0
+plotnine==0.12.4
+ninja==1.11.1.1
+latex==0.7.0
+wandb==0.18.5
+setproctitle==1.3.3
+threadpoolctl==3.2.0
+requests==2.32.3
+pyparsing==3.1.1
+smmap==5.0.1
+pyzmq==23.0.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+matplotlib-inline==0.1.6
+latexcodec==1.0.0
+ipython==8.0.0
+patsy==0.5.3
+contourpy==1.2.0
+multidict==6.1.0
+mizani==0.9.3
+urllib3==2.1.0
+tokenizers==0.20.0
+MarkupSafe==2.1.2
+pip==24.2
+pexpect==4.8.0
+tqdm==4.66.5
+jedi==0.18.2
+pydantic_core==2.23.4
+tempdir==0.7.1
+mpmath==1.2.1
+setuptools==72.1.0
+pytest==7.4.3
+pure-eval==0.2.2
+psutil==5.9.1
+comm==0.1.2
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+regex==2023.10.3
+platformdirs==2.5.2
+asttokens==2.2.1
+torch==2.0.0
+nvidia-nccl-cu11==2.14.3
+xxhash==3.5.0

wandb/run-20241101_201707-vk4mc2rm/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os":  "Linux-5.4.0-162-generic-x86_64-with-glibc2.31",
+  "python":  "3.9.19",
+  "startedAt":  "2024-11-02T00:17:07.952727Z",
+  "args":  [
+    "--perturbation",
+    "shuffle_nondeterministic",
+    "--train_set",
+    "10M",
+    "--batch_size",
+    "3",
+    "--epoch",
+    "3",
+    "--seed",
+    "0"
+  ],
+  "program":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py",
+  "codePath":  "train/train_deep_wandb.py",
+  "git":  {
+    "remote":  "git@hf.co:Yaning1001/Impossible_llm.git",
+    "commit":  "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4"
+  },
+  "email":  "yaning1001@gmail.com",
+  "root":  "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train",
+  "host":  "mms-large-2",
+  "username":  "chunhui",
+  "executable":  "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python",
+  "codePathLocal":  "train_deep_wandb.py",
+  "cpu_count":  32,
+  "cpu_count_logical":  64,
+  "gpu":  "NVIDIA RTX A6000",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1888559353856",
+      "used":  "1754802659328"
+    }
+  },
+  "memory":  {
+    "total":  "202617098240"
+  },
+  "cpu":  {
+    "count":  32,
+    "countLogical":  64
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA RTX A6000",
+      "memoryTotal":  "51527024640",
+      "cudaCores":  10752,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "11.8"
+}

wandb/run-20241101_201707-vk4mc2rm/logs/debug-internal.log

@@ -0,0 +1,8 @@
+{"time":"2024-11-01T20:17:07.955398482-04:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-11-01T20:17:07.955420112-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_201707-vk4mc2rm/logs/debug-core.log"}
+{"time":"2024-11-01T20:17:08.063509154-04:00","level":"INFO","msg":"created new stream","id":"vk4mc2rm"}
+{"time":"2024-11-01T20:17:08.063543045-04:00","level":"INFO","msg":"stream: started","id":"vk4mc2rm"}
+{"time":"2024-11-01T20:17:08.063595545-04:00","level":"INFO","msg":"sender: started","stream_id":"vk4mc2rm"}
+{"time":"2024-11-01T20:17:08.063585335-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"vk4mc2rm"}}
+{"time":"2024-11-01T20:17:08.063567115-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"vk4mc2rm"}}
+{"time":"2024-11-01T20:17:10.018436267-04:00","level":"INFO","msg":"Starting system monitor"}