diff --git a/.gitattributes b/.gitattributes index 360988054fd2a2b662a34f0e3273281c5a006212..f416aca21df581cffe6d198393cb7078d1c4ae39 100644 --- a/.gitattributes +++ b/.gitattributes @@ -39,3 +39,17 @@ wandb/run-20241101_012733-4u8e027p/run-4u8e027p.wandb filter=lfs diff=lfs merge= wandb/run-20241101_012733-e3zsr634/run-e3zsr634.wandb filter=lfs diff=lfs merge=lfs -text wandb/run-20241030_011509-3dp0dtmk/run-3dp0dtmk.wandb filter=lfs diff=lfs merge=lfs -text wandb/run-20241129_235322-dmnv987j/run-dmnv987j.wandb filter=lfs diff=lfs merge=lfs -text +wandb/run-20241105_161832-c18fx9uc/run-c18fx9uc.wandb filter=lfs diff=lfs merge=lfs -text +wandb/run-20241030_011509-46hc4g2h/run-46hc4g2h.wandb filter=lfs diff=lfs merge=lfs -text +wandb/run-20241129_235241-3ftwr60m/run-3ftwr60m.wandb filter=lfs diff=lfs merge=lfs -text +wandb/run-20241031_001055-u3ke9zn3/run-u3ke9zn3.wandb filter=lfs diff=lfs merge=lfs -text +wandb/run-20241113_180154-y8wvn5hq/run-y8wvn5hq.wandb filter=lfs diff=lfs merge=lfs -text +wandb/run-20241030_011509-oh4ul0xu/run-oh4ul0xu.wandb filter=lfs diff=lfs merge=lfs -text +wandb/run-20241030_225833-frh96rd1/run-frh96rd1.wandb filter=lfs diff=lfs merge=lfs -text +wandb/run-20241030_011509-a3dncwci/run-a3dncwci.wandb filter=lfs diff=lfs merge=lfs -text +wandb/run-20241030_013141-bkcoggdw/run-bkcoggdw.wandb filter=lfs diff=lfs merge=lfs -text +wandb/run-20241031_001055-32u9qnul/run-32u9qnul.wandb filter=lfs diff=lfs merge=lfs -text +wandb/run-20241030_231835-i7aqcsql/run-i7aqcsql.wandb filter=lfs diff=lfs merge=lfs -text +wandb/run-20241031_001055-sr4xke8e/run-sr4xke8e.wandb filter=lfs diff=lfs merge=lfs -text +wandb/run-20241031_114700-jx2hqvx3/run-jx2hqvx3.wandb filter=lfs diff=lfs merge=lfs -text +wandb/run-20241105_161832-sl992h9i/run-sl992h9i.wandb filter=lfs diff=lfs merge=lfs -text diff --git a/wandb/run-20241030_011509-46hc4g2h/run-46hc4g2h.wandb b/wandb/run-20241030_011509-46hc4g2h/run-46hc4g2h.wandb new file mode 100644 index 0000000000000000000000000000000000000000..d44198494e8207afc431b2229c04410c2c6a793a --- /dev/null +++ b/wandb/run-20241030_011509-46hc4g2h/run-46hc4g2h.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1168507fc2efdaf39abd2e16fe247ea898936a7fd756db28b36941c1628320dd +size 163840 diff --git a/wandb/run-20241030_011509-a3dncwci/run-a3dncwci.wandb b/wandb/run-20241030_011509-a3dncwci/run-a3dncwci.wandb new file mode 100644 index 0000000000000000000000000000000000000000..5c13e644916198a2626e0d09156fb9c00987c1d7 --- /dev/null +++ b/wandb/run-20241030_011509-a3dncwci/run-a3dncwci.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31de4a80aa64e458b2e817e0bca2f90952a6a1bd55133b08076f86efb73888fe +size 163840 diff --git a/wandb/run-20241030_011509-oh4ul0xu/run-oh4ul0xu.wandb b/wandb/run-20241030_011509-oh4ul0xu/run-oh4ul0xu.wandb new file mode 100644 index 0000000000000000000000000000000000000000..f6dedc0c89e9146b29b70f7e6802bf9c57a3d77b --- /dev/null +++ b/wandb/run-20241030_011509-oh4ul0xu/run-oh4ul0xu.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5cae494d645173c2ab260b7431cfdd4cbe8a6d30ae2635b0edd4afb99e9a3102 +size 163840 diff --git a/wandb/run-20241030_013141-bkcoggdw/run-bkcoggdw.wandb b/wandb/run-20241030_013141-bkcoggdw/run-bkcoggdw.wandb new file mode 100644 index 0000000000000000000000000000000000000000..72e55b9012ba78567fe356b957286be37727214b --- /dev/null +++ b/wandb/run-20241030_013141-bkcoggdw/run-bkcoggdw.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ac456d5195118e26175a05980cc4f5d17ae96370198e1ba04d8c966b27252a0 +size 163840 diff --git a/wandb/run-20241030_225833-frh96rd1/run-frh96rd1.wandb b/wandb/run-20241030_225833-frh96rd1/run-frh96rd1.wandb new file mode 100644 index 0000000000000000000000000000000000000000..6731d4ceaa6f90f795a6704c950f3e8126fba93e --- /dev/null +++ b/wandb/run-20241030_225833-frh96rd1/run-frh96rd1.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70aaf1b9cc1adf99161f62fb97c95548b8cf7b0225e78cc55438924d0f2642cf +size 1015808 diff --git a/wandb/run-20241030_231835-i7aqcsql/run-i7aqcsql.wandb b/wandb/run-20241030_231835-i7aqcsql/run-i7aqcsql.wandb new file mode 100644 index 0000000000000000000000000000000000000000..867c312b79945484872dfed29c0db59936fce4f8 --- /dev/null +++ b/wandb/run-20241030_231835-i7aqcsql/run-i7aqcsql.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0bb6c061238472c5dba05ef529b177a7b24c4546adb8bd6c06a1dc34c306a75 +size 655360 diff --git a/wandb/run-20241030_233740-0fagh7s8/files/output.log b/wandb/run-20241030_233740-0fagh7s8/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..4c5f03f116ff5af53b43549f3eba8d5f95b344fa --- /dev/null +++ b/wandb/run-20241030_233740-0fagh7s8/files/output.log @@ -0,0 +1,190 @@ +Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:05<00:00, 2.60s/it] +tokenized_valid: Dataset({ + features: ['input_ids', 'attention_mask'], + num_rows: 600 +}) +/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +[2024-10-30 23:37:50,273] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-10-30 23:37:58,862] [INFO] [comm.py:652:init_distributed] cdb=None +[2024-10-30 23:37:58,863] [INFO] [comm.py:683:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl +Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher. +Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination +Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root... +Loading extension module cpu_adam... +Time to load cpu_adam op: 4.667487144470215 seconds +wandb: WARNING The `run_name` is currently set to the same value as `TrainingArguments.output_dir`. If this was not intended, please specify a different run name by setting the `TrainingArguments.run_name` parameter. + +{'loss': 2.7522, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.0} +{'loss': 2.7541, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.0} +{'loss': 2.8031, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.0} +{'loss': 2.8062, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.0} +{'loss': 2.7733, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.01} +{'loss': 2.7949, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.01} +{'loss': 2.7657, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.01} +{'loss': 2.803, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.01} +{'loss': 2.7793, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.01} +{'loss': 2.7935, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.01} + +{'eval_loss': 2.812453508377075, 'eval_runtime': 12.4226, 'eval_samples_per_second': 48.299, 'eval_steps_per_second': 1.046, 'epoch': 0.01} +{'loss': 2.7546, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.01} +{'loss': 2.7423, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.01} +{'loss': 2.8057, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.01} +{'loss': 2.7663, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.01} +{'loss': 2.7715, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.02} +{'loss': 2.751, 'grad_norm': 4.9052934646606445, 'learning_rate': 4.998288843258043e-06, 'epoch': 0.02} +{'loss': 2.6716, 'grad_norm': 4.931762218475342, 'learning_rate': 4.996577686516086e-06, 'epoch': 0.02} +{'loss': 2.6561, 'grad_norm': 4.931762218475342, 'learning_rate': 4.996577686516086e-06, 'epoch': 0.02} +{'loss': 2.579, 'grad_norm': 4.839107036590576, 'learning_rate': 4.994866529774127e-06, 'epoch': 0.02} +{'loss': 2.5964, 'grad_norm': 3.662093162536621, 'learning_rate': 4.99315537303217e-06, 'epoch': 0.02} +{'eval_loss': 2.5994045734405518, 'eval_runtime': 11.4634, 'eval_samples_per_second': 52.341, 'eval_steps_per_second': 1.134, 'epoch': 0.02} +{'loss': 2.5669, 'grad_norm': 3.5343265533447266, 'learning_rate': 4.991444216290213e-06, 'epoch': 0.02} +{'loss': 2.5163, 'grad_norm': 2.8458690643310547, 'learning_rate': 4.989733059548255e-06, 'epoch': 0.02} +{'loss': 2.4443, 'grad_norm': 2.5915985107421875, 'learning_rate': 4.988021902806298e-06, 'epoch': 0.02} +{'loss': 2.4325, 'grad_norm': 2.206981658935547, 'learning_rate': 4.9863107460643404e-06, 'epoch': 0.02} +{'loss': 2.3948, 'grad_norm': 2.2773280143737793, 'learning_rate': 4.984599589322382e-06, 'epoch': 0.03} +{'loss': 2.4063, 'grad_norm': 2.1317105293273926, 'learning_rate': 4.982888432580425e-06, 'epoch': 0.03} +{'loss': 2.3657, 'grad_norm': 2.3999195098876953, 'learning_rate': 4.9811772758384674e-06, 'epoch': 0.03} +{'loss': 2.3674, 'grad_norm': 1.9440910816192627, 'learning_rate': 4.97946611909651e-06, 'epoch': 0.03} +{'loss': 2.3386, 'grad_norm': 2.343914747238159, 'learning_rate': 4.977754962354553e-06, 'epoch': 0.03} +{'loss': 2.3157, 'grad_norm': 2.1667869091033936, 'learning_rate': 4.976043805612594e-06, 'epoch': 0.03} +{'eval_loss': 2.3952019214630127, 'eval_runtime': 11.4922, 'eval_samples_per_second': 52.209, 'eval_steps_per_second': 1.131, 'epoch': 0.03} +{'loss': 2.2711, 'grad_norm': 2.118894338607788, 'learning_rate': 4.974332648870637e-06, 'epoch': 0.03} +{'loss': 2.3037, 'grad_norm': 2.242431640625, 'learning_rate': 4.972621492128679e-06, 'epoch': 0.03} +{'loss': 2.2631, 'grad_norm': 1.9315046072006226, 'learning_rate': 4.970910335386721e-06, 'epoch': 0.03} +{'loss': 2.2326, 'grad_norm': 2.2613368034362793, 'learning_rate': 4.969199178644764e-06, 'epoch': 0.03} +{'loss': 2.2002, 'grad_norm': 1.979921579360962, 'learning_rate': 4.967488021902807e-06, 'epoch': 0.04} +{'loss': 2.0944, 'grad_norm': 2.027212381362915, 'learning_rate': 4.965776865160849e-06, 'epoch': 0.04} +{'loss': 2.2204, 'grad_norm': 2.199139356613159, 'learning_rate': 4.964065708418892e-06, 'epoch': 0.04} +{'loss': 2.1221, 'grad_norm': 2.1344027519226074, 'learning_rate': 4.962354551676934e-06, 'epoch': 0.04} +{'loss': 2.1284, 'grad_norm': 2.298335552215576, 'learning_rate': 4.960643394934976e-06, 'epoch': 0.04} +{'loss': 2.1842, 'grad_norm': 2.3745176792144775, 'learning_rate': 4.958932238193019e-06, 'epoch': 0.04} +{'eval_loss': 2.255570650100708, 'eval_runtime': 11.4984, 'eval_samples_per_second': 52.181, 'eval_steps_per_second': 1.131, 'epoch': 0.04} +{'loss': 2.1337, 'grad_norm': 2.2507293224334717, 'learning_rate': 4.9572210814510614e-06, 'epoch': 0.04} +{'loss': 2.0571, 'grad_norm': 2.3284757137298584, 'learning_rate': 4.955509924709104e-06, 'epoch': 0.04} +{'loss': 2.1204, 'grad_norm': 2.172335624694824, 'learning_rate': 4.953798767967146e-06, 'epoch': 0.04} +{'loss': 2.0751, 'grad_norm': 1.9553546905517578, 'learning_rate': 4.952087611225188e-06, 'epoch': 0.05} +{'loss': 2.0911, 'grad_norm': 2.0383987426757812, 'learning_rate': 4.950376454483231e-06, 'epoch': 0.05} +{'loss': 2.06, 'grad_norm': 1.9470635652542114, 'learning_rate': 4.948665297741274e-06, 'epoch': 0.05} +{'loss': 2.1145, 'grad_norm': 1.7316076755523682, 'learning_rate': 4.946954140999316e-06, 'epoch': 0.05} +{'loss': 2.0498, 'grad_norm': 1.9633406400680542, 'learning_rate': 4.945242984257359e-06, 'epoch': 0.05} +{'loss': 2.0609, 'grad_norm': 1.8334592580795288, 'learning_rate': 4.943531827515401e-06, 'epoch': 0.05} +{'loss': 1.988, 'grad_norm': 1.8270950317382812, 'learning_rate': 4.941820670773443e-06, 'epoch': 0.05} +{'eval_loss': 2.1854093074798584, 'eval_runtime': 11.5011, 'eval_samples_per_second': 52.169, 'eval_steps_per_second': 1.13, 'epoch': 0.05} +{'loss': 1.989, 'grad_norm': 1.8450721502304077, 'learning_rate': 4.940109514031486e-06, 'epoch': 0.05} +{'loss': 2.0996, 'grad_norm': 2.2471470832824707, 'learning_rate': 4.9383983572895284e-06, 'epoch': 0.05} +{'loss': 1.9723, 'grad_norm': 1.864260196685791, 'learning_rate': 4.936687200547571e-06, 'epoch': 0.05} +{'loss': 2.0376, 'grad_norm': 2.2607100009918213, 'learning_rate': 4.934976043805613e-06, 'epoch': 0.06} +{'loss': 1.9901, 'grad_norm': 1.9184648990631104, 'learning_rate': 4.9332648870636554e-06, 'epoch': 0.06} +{'loss': 1.9581, 'grad_norm': 1.981023907661438, 'learning_rate': 4.931553730321697e-06, 'epoch': 0.06} +{'loss': 2.0003, 'grad_norm': 2.0862128734588623, 'learning_rate': 4.92984257357974e-06, 'epoch': 0.06} +{'loss': 1.9247, 'grad_norm': 1.8063560724258423, 'learning_rate': 4.928131416837782e-06, 'epoch': 0.06} +{'loss': 1.9957, 'grad_norm': 1.9832228422164917, 'learning_rate': 4.926420260095825e-06, 'epoch': 0.06} +{'loss': 1.9882, 'grad_norm': 2.1227223873138428, 'learning_rate': 4.924709103353868e-06, 'epoch': 0.06} +{'eval_loss': 2.138824462890625, 'eval_runtime': 11.5113, 'eval_samples_per_second': 52.123, 'eval_steps_per_second': 1.129, 'epoch': 0.06} +{'loss': 1.9439, 'grad_norm': 2.0159411430358887, 'learning_rate': 4.92299794661191e-06, 'epoch': 0.06} +{'loss': 1.9647, 'grad_norm': 2.2820119857788086, 'learning_rate': 4.921286789869952e-06, 'epoch': 0.06} +{'loss': 2.012, 'grad_norm': 2.0620205402374268, 'learning_rate': 4.919575633127995e-06, 'epoch': 0.06} +{'loss': 1.9966, 'grad_norm': 2.2670867443084717, 'learning_rate': 4.917864476386037e-06, 'epoch': 0.07} +{'loss': 1.8535, 'grad_norm': 2.0552380084991455, 'learning_rate': 4.91615331964408e-06, 'epoch': 0.07} +{'loss': 1.9662, 'grad_norm': 2.675915241241455, 'learning_rate': 4.9144421629021224e-06, 'epoch': 0.07} +{'loss': 1.9008, 'grad_norm': 2.1373021602630615, 'learning_rate': 4.912731006160164e-06, 'epoch': 0.07} +{'loss': 1.979, 'grad_norm': 2.100844621658325, 'learning_rate': 4.911019849418207e-06, 'epoch': 0.07} +{'loss': 1.9173, 'grad_norm': 2.0223195552825928, 'learning_rate': 4.9093086926762494e-06, 'epoch': 0.07} +{'loss': 2.0353, 'grad_norm': 2.165973663330078, 'learning_rate': 4.907597535934292e-06, 'epoch': 0.07} +{'eval_loss': 2.1140758991241455, 'eval_runtime': 11.5235, 'eval_samples_per_second': 52.068, 'eval_steps_per_second': 1.128, 'epoch': 0.07} +{'loss': 1.9297, 'grad_norm': 2.223146677017212, 'learning_rate': 4.905886379192335e-06, 'epoch': 0.07} +{'loss': 1.8733, 'grad_norm': 2.062178373336792, 'learning_rate': 4.904175222450377e-06, 'epoch': 0.07} +{'loss': 1.9402, 'grad_norm': 2.034820318222046, 'learning_rate': 4.902464065708419e-06, 'epoch': 0.07} +{'loss': 1.9037, 'grad_norm': 2.516028881072998, 'learning_rate': 4.900752908966462e-06, 'epoch': 0.08} +{'loss': 1.9109, 'grad_norm': 2.4920814037323, 'learning_rate': 4.899041752224504e-06, 'epoch': 0.08} +{'loss': 1.9042, 'grad_norm': 2.6503076553344727, 'learning_rate': 4.897330595482547e-06, 'epoch': 0.08} +{'loss': 1.9213, 'grad_norm': 2.3713159561157227, 'learning_rate': 4.8956194387405895e-06, 'epoch': 0.08} +{'loss': 1.9263, 'grad_norm': 2.7794573307037354, 'learning_rate': 4.893908281998632e-06, 'epoch': 0.08} +{'loss': 1.8679, 'grad_norm': 2.1413917541503906, 'learning_rate': 4.892197125256674e-06, 'epoch': 0.08} +{'loss': 1.9298, 'grad_norm': 2.5015697479248047, 'learning_rate': 4.8904859685147164e-06, 'epoch': 0.08} +{'eval_loss': 2.0753817558288574, 'eval_runtime': 11.5583, 'eval_samples_per_second': 51.911, 'eval_steps_per_second': 1.125, 'epoch': 0.08} +{'loss': 1.8715, 'grad_norm': 2.1490447521209717, 'learning_rate': 4.888774811772759e-06, 'epoch': 0.08} +{'loss': 1.9024, 'grad_norm': 2.6189815998077393, 'learning_rate': 4.887063655030802e-06, 'epoch': 0.08} +{'loss': 1.8966, 'grad_norm': 2.212202310562134, 'learning_rate': 4.8853524982888434e-06, 'epoch': 0.09} +{'loss': 1.8856, 'grad_norm': 2.5672545433044434, 'learning_rate': 4.883641341546886e-06, 'epoch': 0.09} +{'loss': 1.854, 'grad_norm': 2.3435440063476562, 'learning_rate': 4.881930184804929e-06, 'epoch': 0.09} +{'loss': 1.9035, 'grad_norm': 2.4554519653320312, 'learning_rate': 4.88021902806297e-06, 'epoch': 0.09} +{'loss': 1.8662, 'grad_norm': 2.377504825592041, 'learning_rate': 4.878507871321013e-06, 'epoch': 0.09} +{'loss': 1.8593, 'grad_norm': 2.3757731914520264, 'learning_rate': 4.876796714579056e-06, 'epoch': 0.09} +{'loss': 1.8779, 'grad_norm': 2.5893397331237793, 'learning_rate': 4.875085557837098e-06, 'epoch': 0.09} +{'loss': 1.8233, 'grad_norm': 2.3579938411712646, 'learning_rate': 4.873374401095141e-06, 'epoch': 0.09} +{'eval_loss': 2.0544168949127197, 'eval_runtime': 11.5311, 'eval_samples_per_second': 52.033, 'eval_steps_per_second': 1.127, 'epoch': 0.09} +{'loss': 1.8527, 'grad_norm': 2.5492780208587646, 'learning_rate': 4.8716632443531835e-06, 'epoch': 0.09} +{'loss': 1.8841, 'grad_norm': 2.0388078689575195, 'learning_rate': 4.869952087611225e-06, 'epoch': 0.09} +{'loss': 1.872, 'grad_norm': 2.2521777153015137, 'learning_rate': 4.868240930869268e-06, 'epoch': 0.1} +{'loss': 1.8546, 'grad_norm': 2.197253704071045, 'learning_rate': 4.8665297741273105e-06, 'epoch': 0.1} +{'loss': 1.9132, 'grad_norm': 2.158266544342041, 'learning_rate': 4.864818617385353e-06, 'epoch': 0.1} +{'loss': 1.7997, 'grad_norm': 2.40034556388855, 'learning_rate': 4.863107460643396e-06, 'epoch': 0.1} +{'loss': 1.874, 'grad_norm': 2.6923694610595703, 'learning_rate': 4.8613963039014374e-06, 'epoch': 0.1} +{'loss': 1.7818, 'grad_norm': 2.381399631500244, 'learning_rate': 4.85968514715948e-06, 'epoch': 0.1} +{'loss': 1.7879, 'grad_norm': 2.354761838912964, 'learning_rate': 4.857973990417523e-06, 'epoch': 0.1} +{'loss': 1.8158, 'grad_norm': 2.516366720199585, 'learning_rate': 4.856262833675565e-06, 'epoch': 0.1} +{'eval_loss': 2.036221981048584, 'eval_runtime': 11.5436, 'eval_samples_per_second': 51.977, 'eval_steps_per_second': 1.126, 'epoch': 0.1} +{'loss': 1.83, 'grad_norm': 2.0844292640686035, 'learning_rate': 4.854551676933608e-06, 'epoch': 0.1} +{'loss': 1.8088, 'grad_norm': 2.376721143722534, 'learning_rate': 4.8528405201916505e-06, 'epoch': 0.1} +{'loss': 1.7779, 'grad_norm': 2.107591390609741, 'learning_rate': 4.851129363449692e-06, 'epoch': 0.11} +{'loss': 1.7956, 'grad_norm': 2.4341471195220947, 'learning_rate': 4.849418206707735e-06, 'epoch': 0.11} +{'loss': 1.8709, 'grad_norm': 2.6235601902008057, 'learning_rate': 4.8477070499657775e-06, 'epoch': 0.11} +{'loss': 1.8434, 'grad_norm': 2.254868984222412, 'learning_rate': 4.84599589322382e-06, 'epoch': 0.11} +{'loss': 1.8517, 'grad_norm': 2.4880223274230957, 'learning_rate': 4.844284736481863e-06, 'epoch': 0.11} +{'loss': 1.8268, 'grad_norm': 2.358351945877075, 'learning_rate': 4.8425735797399045e-06, 'epoch': 0.11} +{'loss': 1.7414, 'grad_norm': 2.4404406547546387, 'learning_rate': 4.840862422997947e-06, 'epoch': 0.11} +{'loss': 1.7884, 'grad_norm': 2.3423702716827393, 'learning_rate': 4.839151266255989e-06, 'epoch': 0.11} +{'eval_loss': 2.0196220874786377, 'eval_runtime': 11.5245, 'eval_samples_per_second': 52.063, 'eval_steps_per_second': 1.128, 'epoch': 0.11} +{'loss': 1.7282, 'grad_norm': 2.115130662918091, 'learning_rate': 4.8374401095140314e-06, 'epoch': 0.11} +{'loss': 1.8166, 'grad_norm': 2.571716547012329, 'learning_rate': 4.835728952772074e-06, 'epoch': 0.11} +{'loss': 1.7803, 'grad_norm': 2.3601763248443604, 'learning_rate': 4.834017796030117e-06, 'epoch': 0.12} +{'loss': 1.7986, 'grad_norm': 2.492929220199585, 'learning_rate': 4.832306639288159e-06, 'epoch': 0.12} +{'loss': 1.7451, 'grad_norm': 2.342839479446411, 'learning_rate': 4.830595482546202e-06, 'epoch': 0.12} +{'loss': 1.8559, 'grad_norm': 2.339629650115967, 'learning_rate': 4.828884325804244e-06, 'epoch': 0.12} +{'loss': 1.788, 'grad_norm': 2.605527639389038, 'learning_rate': 4.827173169062286e-06, 'epoch': 0.12} +{'loss': 1.8042, 'grad_norm': 2.326620578765869, 'learning_rate': 4.825462012320329e-06, 'epoch': 0.12} +{'loss': 1.8078, 'grad_norm': 2.5194342136383057, 'learning_rate': 4.8237508555783715e-06, 'epoch': 0.12} +{'loss': 1.8827, 'grad_norm': 2.5342624187469482, 'learning_rate': 4.822039698836414e-06, 'epoch': 0.12} +{'eval_loss': 2.013911247253418, 'eval_runtime': 11.5397, 'eval_samples_per_second': 51.994, 'eval_steps_per_second': 1.127, 'epoch': 0.12} +{'loss': 1.8128, 'grad_norm': 2.2788140773773193, 'learning_rate': 4.820328542094456e-06, 'epoch': 0.12} +{'loss': 1.7875, 'grad_norm': 2.805044651031494, 'learning_rate': 4.8186173853524985e-06, 'epoch': 0.13} +{'loss': 1.7445, 'grad_norm': 2.242846727371216, 'learning_rate': 4.816906228610541e-06, 'epoch': 0.13} +{'loss': 1.7738, 'grad_norm': 3.2011518478393555, 'learning_rate': 4.815195071868584e-06, 'epoch': 0.13} +{'loss': 1.7282, 'grad_norm': 2.282356023788452, 'learning_rate': 4.813483915126626e-06, 'epoch': 0.13} +{'loss': 1.7981, 'grad_norm': 2.6586058139801025, 'learning_rate': 4.811772758384669e-06, 'epoch': 0.13} +{'loss': 1.7205, 'grad_norm': 2.262554407119751, 'learning_rate': 4.810061601642711e-06, 'epoch': 0.13} +{'loss': 1.7682, 'grad_norm': 2.705533266067505, 'learning_rate': 4.808350444900753e-06, 'epoch': 0.13} +{'loss': 1.8276, 'grad_norm': 2.4971351623535156, 'learning_rate': 4.806639288158796e-06, 'epoch': 0.13} +{'loss': 1.8191, 'grad_norm': 2.391136884689331, 'learning_rate': 4.8049281314168385e-06, 'epoch': 0.13} +{'eval_loss': 1.9947330951690674, 'eval_runtime': 11.5227, 'eval_samples_per_second': 52.071, 'eval_steps_per_second': 1.128, 'epoch': 0.13} +{'loss': 1.7417, 'grad_norm': 2.3191349506378174, 'learning_rate': 4.803216974674881e-06, 'epoch': 0.13} +{'loss': 1.8142, 'grad_norm': 2.5745604038238525, 'learning_rate': 4.801505817932923e-06, 'epoch': 0.14} +{'loss': 1.7814, 'grad_norm': 2.19454026222229, 'learning_rate': 4.7997946611909655e-06, 'epoch': 0.14} +{'loss': 1.7395, 'grad_norm': 2.351425886154175, 'learning_rate': 4.798083504449008e-06, 'epoch': 0.14} +{'loss': 1.7758, 'grad_norm': 2.460624933242798, 'learning_rate': 4.796372347707051e-06, 'epoch': 0.14} +{'loss': 1.7903, 'grad_norm': 2.3103744983673096, 'learning_rate': 4.7946611909650925e-06, 'epoch': 0.14} +{'loss': 1.7159, 'grad_norm': 2.410243272781372, 'learning_rate': 4.792950034223135e-06, 'epoch': 0.14} +{'loss': 1.7891, 'grad_norm': 2.3782598972320557, 'learning_rate': 4.791238877481178e-06, 'epoch': 0.14} +{'loss': 1.7334, 'grad_norm': 2.193119525909424, 'learning_rate': 4.78952772073922e-06, 'epoch': 0.14} +{'loss': 1.7392, 'grad_norm': 2.43776535987854, 'learning_rate': 4.787816563997262e-06, 'epoch': 0.14} +{'eval_loss': 1.9786564111709595, 'eval_runtime': 11.5696, 'eval_samples_per_second': 51.86, 'eval_steps_per_second': 1.124, 'epoch': 0.14} +{'loss': 1.7707, 'grad_norm': 2.404174566268921, 'learning_rate': 4.786105407255305e-06, 'epoch': 0.14} +{'loss': 1.7237, 'grad_norm': 2.301892042160034, 'learning_rate': 4.784394250513347e-06, 'epoch': 0.15} +{'loss': 1.8541, 'grad_norm': 2.5396673679351807, 'learning_rate': 4.78268309377139e-06, 'epoch': 0.15} +{'loss': 1.7333, 'grad_norm': 2.522949457168579, 'learning_rate': 4.7809719370294325e-06, 'epoch': 0.15} +{'loss': 1.7192, 'grad_norm': 2.1934566497802734, 'learning_rate': 4.779260780287474e-06, 'epoch': 0.15} +{'loss': 1.7228, 'grad_norm': 2.566537618637085, 'learning_rate': 4.777549623545517e-06, 'epoch': 0.15} +{'loss': 1.7478, 'grad_norm': 2.3536360263824463, 'learning_rate': 4.7758384668035595e-06, 'epoch': 0.15} +{'loss': 1.7562, 'grad_norm': 2.2443368434906006, 'learning_rate': 4.774127310061602e-06, 'epoch': 0.15} +{'loss': 1.841, 'grad_norm': 2.413248300552368, 'learning_rate': 4.772416153319645e-06, 'epoch': 0.15} +{'loss': 1.7732, 'grad_norm': 2.375981330871582, 'learning_rate': 4.770704996577687e-06, 'epoch': 0.15} +{'eval_loss': 1.982629418373108, 'eval_runtime': 11.522, 'eval_samples_per_second': 52.074, 'eval_steps_per_second': 1.128, 'epoch': 0.15} +{'loss': 1.6788, 'grad_norm': 2.263878107070923, 'learning_rate': 4.768993839835729e-06, 'epoch': 0.16} +{'loss': 1.7174, 'grad_norm': 2.600919723510742, 'learning_rate': 4.767282683093772e-06, 'epoch': 0.16} +{'loss': 1.7688, 'grad_norm': 2.5808844566345215, 'learning_rate': 4.765571526351814e-06, 'epoch': 0.16} +{'loss': 1.7607, 'grad_norm': 2.5976321697235107, 'learning_rate': 4.763860369609857e-06, 'epoch': 0.16} +{'loss': 1.7058, 'grad_norm': 2.2662572860717773, 'learning_rate': 4.7621492128678995e-06, 'epoch': 0.16} +{'loss': 1.682, 'grad_norm': 2.3581748008728027, 'learning_rate': 4.760438056125941e-06, 'epoch': 0.16} +{'loss': 1.739, 'grad_norm': 2.254718780517578, 'learning_rate': 4.758726899383984e-06, 'epoch': 0.16} diff --git a/wandb/run-20241030_233740-0fagh7s8/files/requirements.txt b/wandb/run-20241030_233740-0fagh7s8/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..95a931302e269cc9e4fa5b719b6511f176ee2416 --- /dev/null +++ b/wandb/run-20241030_233740-0fagh7s8/files/requirements.txt @@ -0,0 +1,147 @@ +funcsigs==1.0.2 +sentry-sdk==2.17.0 +multiprocess==0.70.16 +numpy==1.26.2 +pluralizer==1.2.0 +debugpy==1.6.7 +nvidia-cudnn-cu11==8.5.0.96 +deepspeed==0.15.2 +data==0.4 +pandas==2.1.3 +tomli==2.0.1 +charset-normalizer==3.3.2 +attrs==24.2.0 +aiosignal==1.3.1 +fsspec==2023.10.0 +nvidia-cusparse-cu11==11.7.4.91 +zipp==3.12.0 +mypy-extensions==1.0.0 +datasets==3.0.1 +joblib==1.3.2 +hjson==3.1.0 +traitlets==5.7.1 +stack-data==0.6.0 +transformers==4.45.1 +sympy==1.11.1 +Pygments==2.15.0 +docker-pycreds==0.4.0 +dill==0.3.8 +wheel==0.44.0 +prompt-toolkit==3.0.30 +parso==0.8.3 +ipykernel==6.23.1 +pyarrow==17.0.0 +certifi==2023.11.17 +nvidia-cufft-cu11==10.9.0.58 +six==1.16.0 +pydantic==2.9.2 +click==8.1.7 +nest-asyncio==1.5.6 +gmpy2==2.1.0 +matplotlib==3.8.2 +scipy==1.11.4 +typing_extensions==4.12.2 +statsmodels==0.14.0 +huggingface-hub==0.25.0 +frozenlist==1.4.1 +gpustat==1.1.1 +nvidia-nvtx-cu11==11.7.91 +safetensors==0.4.5 +stanza==1.9.2 +decorator==5.1.1 +seaborn==0.13.0 +sentencepiece==0.2.0 +PyYAML==6.0.1 +black==24.8.0 +protobuf==4.25.1 +pickleshare==0.7.5 +peft==0.13.0 +triton==2.0.0 +nvidia-cuda-runtime-cu11==11.7.99 +Jinja2==3.1.2 +nvidia-cusolver-cu11==11.4.0.1 +executing==1.2.0 +jupyter_client==8.1.0 +pluggy==1.3.0 +cmake==3.30.3 +pytz==2023.3.post1 +aiohappyeyeballs==2.4.2 +kiwisolver==1.4.5 +py-cpuinfo==9.0.0 +Pillow==10.1.0 +ptyprocess==0.7.0 +importlib_resources==6.4.5 +GitPython==3.1.43 +importlib-metadata==6.0.0 +iniconfig==2.0.0 +scikit-learn==1.3.2 +exceptiongroup==1.1.0 +networkx==2.8.6 +accelerate==1.0.0 +nltk==3.8.1 +shutilwhich==1.1.0 +fonttools==4.45.1 +future==0.18.3 +aiohttp==3.10.6 +wcwidth==0.2.5 +idna==3.6 +filelock==3.12.2 +pathspec==0.12.1 +jupyter_core==5.1.0 +lit==18.1.8 +nvidia-curand-cu11==10.2.10.91 +nvidia-cublas-cu11==11.10.3.66 +nvidia-ml-py==12.560.30 +msgpack==1.1.0 +python-dateutil==2.8.2 +blessed==1.20.0 +packaging==23.0 +gitdb==4.0.11 +yarl==1.13.0 +emoji==2.8.0 +tzdata==2023.3 +cycler==0.12.1 +tornado==6.2 +backcall==0.2.0 +plotnine==0.12.4 +ninja==1.11.1.1 +latex==0.7.0 +wandb==0.18.5 +setproctitle==1.3.3 +threadpoolctl==3.2.0 +requests==2.32.3 +pyparsing==3.1.1 +smmap==5.0.1 +pyzmq==23.0.0 +async-timeout==4.0.3 +annotated-types==0.7.0 +matplotlib-inline==0.1.6 +latexcodec==1.0.0 +ipython==8.0.0 +patsy==0.5.3 +contourpy==1.2.0 +multidict==6.1.0 +mizani==0.9.3 +urllib3==2.1.0 +tokenizers==0.20.0 +MarkupSafe==2.1.2 +pip==24.2 +pexpect==4.8.0 +tqdm==4.66.5 +jedi==0.18.2 +pydantic_core==2.23.4 +tempdir==0.7.1 +mpmath==1.2.1 +setuptools==72.1.0 +pytest==7.4.3 +pure-eval==0.2.2 +psutil==5.9.1 +comm==0.1.2 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +regex==2023.10.3 +platformdirs==2.5.2 +asttokens==2.2.1 +torch==2.0.0 +nvidia-nccl-cu11==2.14.3 +xxhash==3.5.0 diff --git a/wandb/run-20241030_233740-0fagh7s8/files/wandb-metadata.json b/wandb/run-20241030_233740-0fagh7s8/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..e4067b37c5fdddb0ed1db5b632194b7c2a84524d --- /dev/null +++ b/wandb/run-20241030_233740-0fagh7s8/files/wandb-metadata.json @@ -0,0 +1,97 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-10-31T03:37:40.842788Z", + "args": [ + "--perturbation", + "reverse_control", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "3", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", + "codePath": "train/train_deep_wandb.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_deep_wandb.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA RTX A6000", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888559353856", + "used": "1711065919488" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + } + ], + "cudaVersion": "11.8" +} \ No newline at end of file diff --git a/wandb/run-20241030_233740-0fagh7s8/logs/debug-internal.log b/wandb/run-20241030_233740-0fagh7s8/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..755306ddd756cc18aeed1f647999a609b0762d0d --- /dev/null +++ b/wandb/run-20241030_233740-0fagh7s8/logs/debug-internal.log @@ -0,0 +1,8 @@ +{"time":"2024-10-30T23:37:40.847295918-04:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-10-30T23:37:40.847310969-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_233740-0fagh7s8/logs/debug-core.log"} +{"time":"2024-10-30T23:37:40.9597517-04:00","level":"INFO","msg":"created new stream","id":"0fagh7s8"} +{"time":"2024-10-30T23:37:40.95982373-04:00","level":"INFO","msg":"stream: started","id":"0fagh7s8"} +{"time":"2024-10-30T23:37:40.95985659-04:00","level":"INFO","msg":"sender: started","stream_id":"0fagh7s8"} +{"time":"2024-10-30T23:37:40.959899061-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"0fagh7s8"}} +{"time":"2024-10-30T23:37:40.95984633-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"0fagh7s8"}} +{"time":"2024-10-30T23:37:41.417240986-04:00","level":"INFO","msg":"Starting system monitor"} diff --git a/wandb/run-20241030_233740-0fagh7s8/logs/debug.log b/wandb/run-20241030_233740-0fagh7s8/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..858fcc6cedd7c5e8181ecc5484c06e99422b01a6 --- /dev/null +++ b/wandb/run-20241030_233740-0fagh7s8/logs/debug.log @@ -0,0 +1,29 @@ +2024-10-30 23:37:40,840 INFO MainThread:464533 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-10-30 23:37:40,840 INFO MainThread:464533 [wandb_setup.py:_flush():79] Configure stats pid to 464533 +2024-10-30 23:37:40,840 INFO MainThread:464533 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-10-30 23:37:40,840 INFO MainThread:464533 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-10-30 23:37:40,840 INFO MainThread:464533 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-10-30 23:37:40,840 INFO MainThread:464533 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-10-30 23:37:40,840 INFO MainThread:464533 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'} +2024-10-30 23:37:40,840 INFO MainThread:464533 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-10-30 23:37:40,841 INFO MainThread:464533 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_233740-0fagh7s8/logs/debug.log +2024-10-30 23:37:40,841 INFO MainThread:464533 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_233740-0fagh7s8/logs/debug-internal.log +2024-10-30 23:37:40,841 INFO MainThread:464533 [wandb_init.py:init():621] calling init triggers +2024-10-30 23:37:40,841 INFO MainThread:464533 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-10-30 23:37:40,841 INFO MainThread:464533 [wandb_init.py:init():671] starting backend +2024-10-30 23:37:40,841 INFO MainThread:464533 [wandb_init.py:init():675] sending inform_init request +2024-10-30 23:37:40,842 INFO MainThread:464533 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-10-30 23:37:40,842 INFO MainThread:464533 [wandb_init.py:init():688] backend started and connected +2024-10-30 23:37:40,845 INFO MainThread:464533 [wandb_init.py:init():783] updated telemetry +2024-10-30 23:37:40,872 INFO MainThread:464533 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-10-30 23:37:41,413 INFO MainThread:464533 [wandb_init.py:init():867] starting run threads in backend +2024-10-30 23:37:42,836 INFO MainThread:464533 [wandb_run.py:_console_start():2463] atexit reg +2024-10-30 23:37:42,836 INFO MainThread:464533 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-10-30 23:37:42,836 INFO MainThread:464533 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-10-30 23:37:42,837 INFO MainThread:464533 [wandb_run.py:_redirect():2401] Redirects installed. +2024-10-30 23:37:42,860 INFO MainThread:464533 [wandb_init.py:init():911] run started, returning control to user process +2024-10-30 23:37:42,861 INFO MainThread:464533 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_control', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0} +2024-10-30 23:38:26,525 INFO MainThread:464533 [wandb_run.py:_config_callback():1390] config_cb None None {'vocab_size': 128256, 'max_position_embeddings': 131072, 'hidden_size': 3072, 'intermediate_size': 8192, 'num_hidden_layers': 28, 'num_attention_heads': 24, 'num_key_value_heads': 8, 'hidden_act': 'silu', 'initializer_range': 0.02, 'rms_norm_eps': 1e-05, 'pretraining_tp': 1, 'use_cache': True, 'rope_theta': 500000.0, 'rope_scaling': {'factor': 32.0, 'high_freq_factor': 4.0, 'low_freq_factor': 1.0, 'original_max_position_embeddings': 8192, 'rope_type': 'llama3'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 128, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'bfloat16', 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': True, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['LlamaForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 128000, 'pad_token_id': None, 'eos_token_id': 128001, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'meta-llama/Llama-3.2-3B', 'transformers_version': '4.45.1', 'model_type': 'llama', 'output_dir': './checkpoints/Llama-3.2-3B/babylm_reverse_control_10M_seed0/runs', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': True, 'do_predict': False, 'eval_strategy': 'steps', 'prediction_loss_only': False, 'per_device_train_batch_size': 3, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'eval_delay': 0, 'torch_empty_cache_steps': None, 'learning_rate': 5e-06, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 3, 'max_steps': -1, 'lr_scheduler_type': 'linear', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.0, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': './logs', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 1, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 150, 'save_total_limit': None, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 0, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': False, 'fp16': True, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 10, 'dataloader_num_workers': 0, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': './checkpoints/Llama-3.2-3B/babylm_reverse_control_10M_seed0/runs', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': 'deepspeed_config/train_dp_config.json', 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'evaluation_strategy': 'steps', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False, 'use_liger_kernel': False, 'eval_use_gather_object': False} +2024-10-30 23:38:26,527 INFO MainThread:464533 [wandb_config.py:__setitem__():154] config set model/num_parameters = 3212749824 - > +2024-10-30 23:38:26,527 INFO MainThread:464533 [wandb_run.py:_config_callback():1390] config_cb model/num_parameters 3212749824 None diff --git a/wandb/run-20241031_001055-32u9qnul/run-32u9qnul.wandb b/wandb/run-20241031_001055-32u9qnul/run-32u9qnul.wandb new file mode 100644 index 0000000000000000000000000000000000000000..10d9e0cef5ba768f538460f8a91068923a7c450e --- /dev/null +++ b/wandb/run-20241031_001055-32u9qnul/run-32u9qnul.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f49988f8aae635634599adabf52569773e38cf954305e4c2a3d38c8e8c46c83 +size 327680 diff --git a/wandb/run-20241031_001055-sr4xke8e/run-sr4xke8e.wandb b/wandb/run-20241031_001055-sr4xke8e/run-sr4xke8e.wandb new file mode 100644 index 0000000000000000000000000000000000000000..48fff7fd11d2431c9aa195c93aa3fbf884d0a6c3 --- /dev/null +++ b/wandb/run-20241031_001055-sr4xke8e/run-sr4xke8e.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:155772d9839e7a89851871c99ac7de107a6aff85919294aa49fe9a729aa05b42 +size 327680 diff --git a/wandb/run-20241031_001055-u3ke9zn3/run-u3ke9zn3.wandb b/wandb/run-20241031_001055-u3ke9zn3/run-u3ke9zn3.wandb new file mode 100644 index 0000000000000000000000000000000000000000..c73d08ed8910ccf0961bc5c7707037b0f5684903 --- /dev/null +++ b/wandb/run-20241031_001055-u3ke9zn3/run-u3ke9zn3.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2e42dc976478e26e3507c297f038b31d00be976a0b3e1b6f4bd2c1bfc0dbbef +size 327680 diff --git a/wandb/run-20241031_114700-jx2hqvx3/run-jx2hqvx3.wandb b/wandb/run-20241031_114700-jx2hqvx3/run-jx2hqvx3.wandb new file mode 100644 index 0000000000000000000000000000000000000000..707ba6dc724817f338d645d85a4f27e147e23070 --- /dev/null +++ b/wandb/run-20241031_114700-jx2hqvx3/run-jx2hqvx3.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48b33ce646e8a0f568b819405b479cdfeccc2daae28acd906189efa464154fd6 +size 1015808 diff --git a/wandb/run-20241031_122113-yzubvl65/files/config.yaml b/wandb/run-20241031_122113-yzubvl65/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..edfa239fe9670ab026a3223bac576877fd80297e --- /dev/null +++ b/wandb/run-20241031_122113-yzubvl65/files/config.yaml @@ -0,0 +1,50 @@ +_wandb: + value: + cli_version: 0.18.5 + m: [] + python_version: 3.9.19 + t: + "1": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "2": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "3": + - 2 + - 13 + - 23 + - 55 + "4": 3.9.19 + "5": 0.18.5 + "6": 4.45.1 + "8": + - 5 + "12": 0.18.5 + "13": linux-x86_64 +batch_size: + value: 3 +epoch: + value: 6 +lr: + value: 5e-06 +perturbation: + value: reverse_full +seed: + value: 0 +train_set: + value: 10M diff --git a/wandb/run-20241031_122113-yzubvl65/files/output.log b/wandb/run-20241031_122113-yzubvl65/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..c449045ac0bd6a19df587a1683a8c328a4fa76f0 --- /dev/null +++ b/wandb/run-20241031_122113-yzubvl65/files/output.log @@ -0,0 +1,15 @@ +model-00001-of-00002.safetensors: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████| 4.97G/4.97G [01:34<00:00, 42.2MB/s] +Downloading shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [02:09<00:00, 64.69s/it] +Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:05<00:00, 2.57s/it] +tokenized_valid: Dataset({ + features: ['input_ids', 'attention_mask'], + num_rows: 600 +}) +/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +[2024-10-31 12:23:30,557] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-10-31 12:23:39,160] [INFO] [comm.py:652:init_distributed] cdb=None +Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination +Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root... +Loading extension module cpu_adam... +Time to load cpu_adam op: 5.0282368659973145 seconds diff --git a/wandb/run-20241031_122113-yzubvl65/files/wandb-metadata.json b/wandb/run-20241031_122113-yzubvl65/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..8e3049b99bb8b6594b7a38b392a985b9f1a50012 --- /dev/null +++ b/wandb/run-20241031_122113-yzubvl65/files/wandb-metadata.json @@ -0,0 +1,97 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-10-31T16:21:13.915797Z", + "args": [ + "--perturbation", + "reverse_full", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "6", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", + "codePath": "train/train_deep_wandb.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_deep_wandb.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA RTX A6000", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888559353856", + "used": "1753159962624" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + } + ], + "cudaVersion": "11.8" +} \ No newline at end of file diff --git a/wandb/run-20241031_122113-yzubvl65/files/wandb-summary.json b/wandb/run-20241031_122113-yzubvl65/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..f78ea47926dfc1c734e02453aa9e6ce3692f2be4 --- /dev/null +++ b/wandb/run-20241031_122113-yzubvl65/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":32015}} \ No newline at end of file diff --git a/wandb/run-20241101_092804-l47v8jb3/files/output.log b/wandb/run-20241101_092804-l47v8jb3/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..cb2862032ae5f3d9467be68ec794694ce1db01f6 --- /dev/null +++ b/wandb/run-20241101_092804-l47v8jb3/files/output.log @@ -0,0 +1,17 @@ +Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:14<00:00, 7.41s/it] +Map: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 17013/17013 [00:47<00:00, 360.48 examples/s] +tokenized_valid: Dataset({ + features: ['input_ids', 'attention_mask'], + num_rows: 600 +}) +/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +[2024-11-01 09:29:08,573] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-11-01 09:29:18,017] [INFO] [comm.py:652:init_distributed] cdb=None +Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination +Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root... +Emitting ninja build file /home/chunhui/.cache/torch_extensions/py39_cu117/cpu_adam/build.ninja... +Building extension module cpu_adam... +Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +Loading extension module cpu_adam... +Time to load cpu_adam op: 5.2360241413116455 seconds diff --git a/wandb/run-20241101_092804-l47v8jb3/files/requirements.txt b/wandb/run-20241101_092804-l47v8jb3/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..95a931302e269cc9e4fa5b719b6511f176ee2416 --- /dev/null +++ b/wandb/run-20241101_092804-l47v8jb3/files/requirements.txt @@ -0,0 +1,147 @@ +funcsigs==1.0.2 +sentry-sdk==2.17.0 +multiprocess==0.70.16 +numpy==1.26.2 +pluralizer==1.2.0 +debugpy==1.6.7 +nvidia-cudnn-cu11==8.5.0.96 +deepspeed==0.15.2 +data==0.4 +pandas==2.1.3 +tomli==2.0.1 +charset-normalizer==3.3.2 +attrs==24.2.0 +aiosignal==1.3.1 +fsspec==2023.10.0 +nvidia-cusparse-cu11==11.7.4.91 +zipp==3.12.0 +mypy-extensions==1.0.0 +datasets==3.0.1 +joblib==1.3.2 +hjson==3.1.0 +traitlets==5.7.1 +stack-data==0.6.0 +transformers==4.45.1 +sympy==1.11.1 +Pygments==2.15.0 +docker-pycreds==0.4.0 +dill==0.3.8 +wheel==0.44.0 +prompt-toolkit==3.0.30 +parso==0.8.3 +ipykernel==6.23.1 +pyarrow==17.0.0 +certifi==2023.11.17 +nvidia-cufft-cu11==10.9.0.58 +six==1.16.0 +pydantic==2.9.2 +click==8.1.7 +nest-asyncio==1.5.6 +gmpy2==2.1.0 +matplotlib==3.8.2 +scipy==1.11.4 +typing_extensions==4.12.2 +statsmodels==0.14.0 +huggingface-hub==0.25.0 +frozenlist==1.4.1 +gpustat==1.1.1 +nvidia-nvtx-cu11==11.7.91 +safetensors==0.4.5 +stanza==1.9.2 +decorator==5.1.1 +seaborn==0.13.0 +sentencepiece==0.2.0 +PyYAML==6.0.1 +black==24.8.0 +protobuf==4.25.1 +pickleshare==0.7.5 +peft==0.13.0 +triton==2.0.0 +nvidia-cuda-runtime-cu11==11.7.99 +Jinja2==3.1.2 +nvidia-cusolver-cu11==11.4.0.1 +executing==1.2.0 +jupyter_client==8.1.0 +pluggy==1.3.0 +cmake==3.30.3 +pytz==2023.3.post1 +aiohappyeyeballs==2.4.2 +kiwisolver==1.4.5 +py-cpuinfo==9.0.0 +Pillow==10.1.0 +ptyprocess==0.7.0 +importlib_resources==6.4.5 +GitPython==3.1.43 +importlib-metadata==6.0.0 +iniconfig==2.0.0 +scikit-learn==1.3.2 +exceptiongroup==1.1.0 +networkx==2.8.6 +accelerate==1.0.0 +nltk==3.8.1 +shutilwhich==1.1.0 +fonttools==4.45.1 +future==0.18.3 +aiohttp==3.10.6 +wcwidth==0.2.5 +idna==3.6 +filelock==3.12.2 +pathspec==0.12.1 +jupyter_core==5.1.0 +lit==18.1.8 +nvidia-curand-cu11==10.2.10.91 +nvidia-cublas-cu11==11.10.3.66 +nvidia-ml-py==12.560.30 +msgpack==1.1.0 +python-dateutil==2.8.2 +blessed==1.20.0 +packaging==23.0 +gitdb==4.0.11 +yarl==1.13.0 +emoji==2.8.0 +tzdata==2023.3 +cycler==0.12.1 +tornado==6.2 +backcall==0.2.0 +plotnine==0.12.4 +ninja==1.11.1.1 +latex==0.7.0 +wandb==0.18.5 +setproctitle==1.3.3 +threadpoolctl==3.2.0 +requests==2.32.3 +pyparsing==3.1.1 +smmap==5.0.1 +pyzmq==23.0.0 +async-timeout==4.0.3 +annotated-types==0.7.0 +matplotlib-inline==0.1.6 +latexcodec==1.0.0 +ipython==8.0.0 +patsy==0.5.3 +contourpy==1.2.0 +multidict==6.1.0 +mizani==0.9.3 +urllib3==2.1.0 +tokenizers==0.20.0 +MarkupSafe==2.1.2 +pip==24.2 +pexpect==4.8.0 +tqdm==4.66.5 +jedi==0.18.2 +pydantic_core==2.23.4 +tempdir==0.7.1 +mpmath==1.2.1 +setuptools==72.1.0 +pytest==7.4.3 +pure-eval==0.2.2 +psutil==5.9.1 +comm==0.1.2 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +regex==2023.10.3 +platformdirs==2.5.2 +asttokens==2.2.1 +torch==2.0.0 +nvidia-nccl-cu11==2.14.3 +xxhash==3.5.0 diff --git a/wandb/run-20241101_092804-l47v8jb3/files/wandb-metadata.json b/wandb/run-20241101_092804-l47v8jb3/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..28be00e2855ead2bbec170bd8132551b7fb1f9c5 --- /dev/null +++ b/wandb/run-20241101_092804-l47v8jb3/files/wandb-metadata.json @@ -0,0 +1,97 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-11-01T13:28:04.307193Z", + "args": [ + "--perturbation", + "shuffle_nondeterministic", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "6", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", + "codePath": "train/train_deep_wandb.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_deep_wandb.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA RTX A6000", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888559353856", + "used": "1754628071424" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + } + ], + "cudaVersion": "11.8" +} \ No newline at end of file diff --git a/wandb/run-20241101_092804-l47v8jb3/logs/debug-internal.log b/wandb/run-20241101_092804-l47v8jb3/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..639f1ea10be8d4838c4c4900aa6b84bb6d840062 --- /dev/null +++ b/wandb/run-20241101_092804-l47v8jb3/logs/debug-internal.log @@ -0,0 +1,8 @@ +{"time":"2024-11-01T09:28:04.310194036-04:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-11-01T09:28:04.310206806-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_092804-l47v8jb3/logs/debug-core.log"} +{"time":"2024-11-01T09:28:04.417666825-04:00","level":"INFO","msg":"created new stream","id":"l47v8jb3"} +{"time":"2024-11-01T09:28:04.417704745-04:00","level":"INFO","msg":"stream: started","id":"l47v8jb3"} +{"time":"2024-11-01T09:28:04.417757526-04:00","level":"INFO","msg":"sender: started","stream_id":"l47v8jb3"} +{"time":"2024-11-01T09:28:04.417744946-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"l47v8jb3"}} +{"time":"2024-11-01T09:28:04.417757796-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"l47v8jb3"}} +{"time":"2024-11-01T09:28:04.604158521-04:00","level":"INFO","msg":"Starting system monitor"} diff --git a/wandb/run-20241101_092804-l47v8jb3/logs/debug.log b/wandb/run-20241101_092804-l47v8jb3/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..884f856d9808c351f93b222fa51b5eff7fe77344 --- /dev/null +++ b/wandb/run-20241101_092804-l47v8jb3/logs/debug.log @@ -0,0 +1,26 @@ +2024-11-01 09:28:04,304 INFO MainThread:779818 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-11-01 09:28:04,304 INFO MainThread:779818 [wandb_setup.py:_flush():79] Configure stats pid to 779818 +2024-11-01 09:28:04,304 INFO MainThread:779818 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-11-01 09:28:04,304 INFO MainThread:779818 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-11-01 09:28:04,305 INFO MainThread:779818 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-01 09:28:04,305 INFO MainThread:779818 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-01 09:28:04,305 INFO MainThread:779818 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'} +2024-11-01 09:28:04,305 INFO MainThread:779818 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-01 09:28:04,305 INFO MainThread:779818 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_092804-l47v8jb3/logs/debug.log +2024-11-01 09:28:04,305 INFO MainThread:779818 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_092804-l47v8jb3/logs/debug-internal.log +2024-11-01 09:28:04,305 INFO MainThread:779818 [wandb_init.py:init():621] calling init triggers +2024-11-01 09:28:04,305 INFO MainThread:779818 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-11-01 09:28:04,305 INFO MainThread:779818 [wandb_init.py:init():671] starting backend +2024-11-01 09:28:04,305 INFO MainThread:779818 [wandb_init.py:init():675] sending inform_init request +2024-11-01 09:28:04,306 INFO MainThread:779818 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-01 09:28:04,307 INFO MainThread:779818 [wandb_init.py:init():688] backend started and connected +2024-11-01 09:28:04,311 INFO MainThread:779818 [wandb_init.py:init():783] updated telemetry +2024-11-01 09:28:04,339 INFO MainThread:779818 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-11-01 09:28:04,601 INFO MainThread:779818 [wandb_init.py:init():867] starting run threads in backend +2024-11-01 09:28:04,698 INFO MainThread:779818 [wandb_run.py:_console_start():2463] atexit reg +2024-11-01 09:28:04,698 INFO MainThread:779818 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-11-01 09:28:04,698 INFO MainThread:779818 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-11-01 09:28:04,698 INFO MainThread:779818 [wandb_run.py:_redirect():2401] Redirects installed. +2024-11-01 09:28:04,700 INFO MainThread:779818 [wandb_init.py:init():911] run started, returning control to user process +2024-11-01 09:28:04,700 INFO MainThread:779818 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_nondeterministic', 'train_set': '10M', 'batch_size': 3, 'epoch': 6, 'seed': 0, 'lr': 5e-06} diff --git a/wandb/run-20241101_092804-l47v8jb3/run-l47v8jb3.wandb b/wandb/run-20241101_092804-l47v8jb3/run-l47v8jb3.wandb new file mode 100644 index 0000000000000000000000000000000000000000..9fd50b2cd12d6899842f5b5a1dbc0fc1f412049c Binary files /dev/null and b/wandb/run-20241101_092804-l47v8jb3/run-l47v8jb3.wandb differ diff --git a/wandb/run-20241101_200502-0isq1nei/files/output.log b/wandb/run-20241101_200502-0isq1nei/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..04ca77dae62c42467f2e3713ee4aa0dba0d01be1 --- /dev/null +++ b/wandb/run-20241101_200502-0isq1nei/files/output.log @@ -0,0 +1 @@ +Loading checkpoint shards: 0%| | 0/2 [00:00 + dataset = load_dataset('babylm_dataset_test.py', name=dataset_name, trust_remote_code=True) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/load.py", line 2074, in load_dataset + builder_instance = load_dataset_builder( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/load.py", line 1832, in load_dataset_builder + builder_instance: DatasetBuilder = builder_cls( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/builder.py", line 342, in __init__ + self.config, self.config_id = self._create_builder_config( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/builder.py", line 569, in _create_builder_config + raise ValueError( +ValueError: BuilderConfig 'babylm_shuffle_nodeterministic_10M_seed0' not found. Available: ['babylm_hop_control_10M_seed0', 'babylm_hop_tokens4_10M_seed0', 'babylm_hop_words4_10M_seed0', 'babylm_reverse_control_10M_seed0', 'babylm_reverse_partial_10M_seed0', 'babylm_reverse_full_10M_seed0', 'babylm_shuffle_control_10M_seed0', 'babylm_shuffle_nondeterministic_10M_seed0', 'babylm_shuffle_deterministic21_10M_seed0', 'babylm_shuffle_deterministic57_10M_seed0', 'babylm_shuffle_deterministic84_10M_seed0', 'babylm_shuffle_local3_10M_seed0', 'babylm_shuffle_local5_10M_seed0', 'babylm_shuffle_local10_10M_seed0', 'babylm_shuffle_even_odd_10M_seed0'] diff --git a/wandb/run-20241101_201630-85vocaw4/files/wandb-metadata.json b/wandb/run-20241101_201630-85vocaw4/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..465758816f5053c0ef8c96a5b98540050718532f --- /dev/null +++ b/wandb/run-20241101_201630-85vocaw4/files/wandb-metadata.json @@ -0,0 +1,29 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-11-02T00:16:30.588147Z", + "args": [ + "--perturbation", + "shuffle_nodeterministic", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "6", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", + "codePath": "train/train_deep_wandb.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_deep_wandb.py" +} \ No newline at end of file diff --git a/wandb/run-20241101_201630-85vocaw4/files/wandb-summary.json b/wandb/run-20241101_201630-85vocaw4/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..6c37fe1cbbb8aed86fd461a79642cb991e4d35cf --- /dev/null +++ b/wandb/run-20241101_201630-85vocaw4/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":0}} \ No newline at end of file diff --git a/wandb/run-20241101_201630-85vocaw4/logs/debug-internal.log b/wandb/run-20241101_201630-85vocaw4/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..0ac2d6feb3d0047eb242d9cc0bb29f1e73777d2e --- /dev/null +++ b/wandb/run-20241101_201630-85vocaw4/logs/debug-internal.log @@ -0,0 +1,16 @@ +{"time":"2024-11-01T20:16:30.590081773-04:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-11-01T20:16:30.590096543-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_201630-85vocaw4/logs/debug-core.log"} +{"time":"2024-11-01T20:16:30.698509196-04:00","level":"INFO","msg":"created new stream","id":"85vocaw4"} +{"time":"2024-11-01T20:16:30.698571246-04:00","level":"INFO","msg":"stream: started","id":"85vocaw4"} +{"time":"2024-11-01T20:16:30.698659457-04:00","level":"INFO","msg":"sender: started","stream_id":"85vocaw4"} +{"time":"2024-11-01T20:16:30.698637567-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"85vocaw4"}} +{"time":"2024-11-01T20:16:30.698633617-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"85vocaw4"}} +{"time":"2024-11-01T20:16:30.872065521-04:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-11-01T20:16:30.969121491-04:00","level":"INFO","msg":"stream: closing","id":"85vocaw4"} +{"time":"2024-11-01T20:16:30.969183031-04:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2024-11-01T20:16:30.977041665-04:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2024-11-01T20:16:31.508663323-04:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2024-11-01T20:16:31.620535336-04:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"85vocaw4"}} +{"time":"2024-11-01T20:16:31.620624207-04:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"85vocaw4"}} +{"time":"2024-11-01T20:16:31.620662317-04:00","level":"INFO","msg":"sender: closed","stream_id":"85vocaw4"} +{"time":"2024-11-01T20:16:31.620703757-04:00","level":"INFO","msg":"stream: closed","id":"85vocaw4"} diff --git a/wandb/run-20241101_201630-85vocaw4/logs/debug.log b/wandb/run-20241101_201630-85vocaw4/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..aefdb554d398fad3b3d93bc67ed4197b36a53513 --- /dev/null +++ b/wandb/run-20241101_201630-85vocaw4/logs/debug.log @@ -0,0 +1,27 @@ +2024-11-01 20:16:30,586 INFO MainThread:874719 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-11-01 20:16:30,586 INFO MainThread:874719 [wandb_setup.py:_flush():79] Configure stats pid to 874719 +2024-11-01 20:16:30,586 INFO MainThread:874719 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-11-01 20:16:30,586 INFO MainThread:874719 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-11-01 20:16:30,586 INFO MainThread:874719 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-01 20:16:30,586 INFO MainThread:874719 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-01 20:16:30,586 INFO MainThread:874719 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'} +2024-11-01 20:16:30,586 INFO MainThread:874719 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-01 20:16:30,586 INFO MainThread:874719 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_201630-85vocaw4/logs/debug.log +2024-11-01 20:16:30,586 INFO MainThread:874719 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_201630-85vocaw4/logs/debug-internal.log +2024-11-01 20:16:30,586 INFO MainThread:874719 [wandb_init.py:init():621] calling init triggers +2024-11-01 20:16:30,586 INFO MainThread:874719 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-11-01 20:16:30,586 INFO MainThread:874719 [wandb_init.py:init():671] starting backend +2024-11-01 20:16:30,586 INFO MainThread:874719 [wandb_init.py:init():675] sending inform_init request +2024-11-01 20:16:30,587 INFO MainThread:874719 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-01 20:16:30,588 INFO MainThread:874719 [wandb_init.py:init():688] backend started and connected +2024-11-01 20:16:30,591 INFO MainThread:874719 [wandb_init.py:init():783] updated telemetry +2024-11-01 20:16:30,625 INFO MainThread:874719 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-11-01 20:16:30,868 INFO MainThread:874719 [wandb_init.py:init():867] starting run threads in backend +2024-11-01 20:16:30,958 INFO MainThread:874719 [wandb_run.py:_console_start():2463] atexit reg +2024-11-01 20:16:30,958 INFO MainThread:874719 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-11-01 20:16:30,958 INFO MainThread:874719 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-11-01 20:16:30,958 INFO MainThread:874719 [wandb_run.py:_redirect():2401] Redirects installed. +2024-11-01 20:16:30,960 INFO MainThread:874719 [wandb_init.py:init():911] run started, returning control to user process +2024-11-01 20:16:30,961 INFO MainThread:874719 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_nodeterministic', 'train_set': '10M', 'batch_size': 3, 'epoch': 6, 'seed': 0, 'lr': 5e-06} +2024-11-01 20:16:30,969 WARNING MsgRouterThr:874719 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20241101_201630-85vocaw4/run-85vocaw4.wandb b/wandb/run-20241101_201630-85vocaw4/run-85vocaw4.wandb new file mode 100644 index 0000000000000000000000000000000000000000..2f923d1cd74a7204308dff7edc00f61ad4612405 Binary files /dev/null and b/wandb/run-20241101_201630-85vocaw4/run-85vocaw4.wandb differ diff --git a/wandb/run-20241101_201630-cb0m59vi/files/config.yaml b/wandb/run-20241101_201630-cb0m59vi/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16d25ee0e4092474360045faaf0959cfc1b3e91d --- /dev/null +++ b/wandb/run-20241101_201630-cb0m59vi/files/config.yaml @@ -0,0 +1,49 @@ +_wandb: + value: + cli_version: 0.18.5 + m: [] + python_version: 3.9.19 + t: + "1": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "2": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "3": + - 13 + - 23 + - 55 + "4": 3.9.19 + "5": 0.18.5 + "6": 4.45.1 + "8": + - 5 + "12": 0.18.5 + "13": linux-x86_64 +batch_size: + value: 3 +epoch: + value: 6 +lr: + value: 5e-06 +perturbation: + value: shuffle_nodeterministic +seed: + value: 0 +train_set: + value: 10M diff --git a/wandb/run-20241101_201630-cb0m59vi/files/output.log b/wandb/run-20241101_201630-cb0m59vi/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..5824b3c1b0e7410256b3374a200db5bd3cc11c9d --- /dev/null +++ b/wandb/run-20241101_201630-cb0m59vi/files/output.log @@ -0,0 +1,12 @@ +Traceback (most recent call last): + File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 164, in + dataset = load_dataset('babylm_dataset_test.py', name=dataset_name, trust_remote_code=True) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/load.py", line 2074, in load_dataset + builder_instance = load_dataset_builder( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/load.py", line 1832, in load_dataset_builder + builder_instance: DatasetBuilder = builder_cls( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/builder.py", line 342, in __init__ + self.config, self.config_id = self._create_builder_config( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/builder.py", line 569, in _create_builder_config + raise ValueError( +ValueError: BuilderConfig 'babylm_shuffle_nodeterministic_10M_seed0' not found. Available: ['babylm_hop_control_10M_seed0', 'babylm_hop_tokens4_10M_seed0', 'babylm_hop_words4_10M_seed0', 'babylm_reverse_control_10M_seed0', 'babylm_reverse_partial_10M_seed0', 'babylm_reverse_full_10M_seed0', 'babylm_shuffle_control_10M_seed0', 'babylm_shuffle_nondeterministic_10M_seed0', 'babylm_shuffle_deterministic21_10M_seed0', 'babylm_shuffle_deterministic57_10M_seed0', 'babylm_shuffle_deterministic84_10M_seed0', 'babylm_shuffle_local3_10M_seed0', 'babylm_shuffle_local5_10M_seed0', 'babylm_shuffle_local10_10M_seed0', 'babylm_shuffle_even_odd_10M_seed0'] diff --git a/wandb/run-20241101_201630-cb0m59vi/files/wandb-metadata.json b/wandb/run-20241101_201630-cb0m59vi/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..121bb14440ee560ce40b02ab89154da8e8bf66e9 --- /dev/null +++ b/wandb/run-20241101_201630-cb0m59vi/files/wandb-metadata.json @@ -0,0 +1,97 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-11-02T00:16:30.561525Z", + "args": [ + "--perturbation", + "shuffle_nodeterministic", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "6", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", + "codePath": "train/train_deep_wandb.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_deep_wandb.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA RTX A6000", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888559353856", + "used": "1754802601984" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + } + ], + "cudaVersion": "11.8" +} \ No newline at end of file diff --git a/wandb/run-20241101_201630-cb0m59vi/files/wandb-summary.json b/wandb/run-20241101_201630-cb0m59vi/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..6c37fe1cbbb8aed86fd461a79642cb991e4d35cf --- /dev/null +++ b/wandb/run-20241101_201630-cb0m59vi/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":0}} \ No newline at end of file diff --git a/wandb/run-20241101_201630-cb0m59vi/logs/debug-internal.log b/wandb/run-20241101_201630-cb0m59vi/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..075e84c7c029a07fb3c32f013c4980456224d67f --- /dev/null +++ b/wandb/run-20241101_201630-cb0m59vi/logs/debug-internal.log @@ -0,0 +1,16 @@ +{"time":"2024-11-01T20:16:30.563400835-04:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-11-01T20:16:30.563412425-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_201630-cb0m59vi/logs/debug-core.log"} +{"time":"2024-11-01T20:16:30.671139114-04:00","level":"INFO","msg":"created new stream","id":"cb0m59vi"} +{"time":"2024-11-01T20:16:30.671176224-04:00","level":"INFO","msg":"stream: started","id":"cb0m59vi"} +{"time":"2024-11-01T20:16:30.671275055-04:00","level":"INFO","msg":"sender: started","stream_id":"cb0m59vi"} +{"time":"2024-11-01T20:16:30.671219504-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"cb0m59vi"}} +{"time":"2024-11-01T20:16:30.671207154-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"cb0m59vi"}} +{"time":"2024-11-01T20:16:30.839933803-04:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-11-01T20:16:30.936897352-04:00","level":"INFO","msg":"stream: closing","id":"cb0m59vi"} +{"time":"2024-11-01T20:16:30.936931672-04:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2024-11-01T20:16:30.937377334-04:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2024-11-01T20:16:31.485218533-04:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2024-11-01T20:16:31.608476399-04:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"cb0m59vi"}} +{"time":"2024-11-01T20:16:31.608528239-04:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"cb0m59vi"}} +{"time":"2024-11-01T20:16:31.608545179-04:00","level":"INFO","msg":"sender: closed","stream_id":"cb0m59vi"} +{"time":"2024-11-01T20:16:31.608580439-04:00","level":"INFO","msg":"stream: closed","id":"cb0m59vi"} diff --git a/wandb/run-20241101_201630-cb0m59vi/logs/debug.log b/wandb/run-20241101_201630-cb0m59vi/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..dd089171fe6e24df75b999c807f6deff17f6139a --- /dev/null +++ b/wandb/run-20241101_201630-cb0m59vi/logs/debug.log @@ -0,0 +1,27 @@ +2024-11-01 20:16:30,559 INFO MainThread:874714 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-11-01 20:16:30,559 INFO MainThread:874714 [wandb_setup.py:_flush():79] Configure stats pid to 874714 +2024-11-01 20:16:30,559 INFO MainThread:874714 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-11-01 20:16:30,559 INFO MainThread:874714 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-11-01 20:16:30,559 INFO MainThread:874714 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-01 20:16:30,559 INFO MainThread:874714 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-01 20:16:30,559 INFO MainThread:874714 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'} +2024-11-01 20:16:30,559 INFO MainThread:874714 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-01 20:16:30,559 INFO MainThread:874714 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_201630-cb0m59vi/logs/debug.log +2024-11-01 20:16:30,559 INFO MainThread:874714 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_201630-cb0m59vi/logs/debug-internal.log +2024-11-01 20:16:30,559 INFO MainThread:874714 [wandb_init.py:init():621] calling init triggers +2024-11-01 20:16:30,559 INFO MainThread:874714 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-11-01 20:16:30,559 INFO MainThread:874714 [wandb_init.py:init():671] starting backend +2024-11-01 20:16:30,559 INFO MainThread:874714 [wandb_init.py:init():675] sending inform_init request +2024-11-01 20:16:30,560 INFO MainThread:874714 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-01 20:16:30,561 INFO MainThread:874714 [wandb_init.py:init():688] backend started and connected +2024-11-01 20:16:30,565 INFO MainThread:874714 [wandb_init.py:init():783] updated telemetry +2024-11-01 20:16:30,586 INFO MainThread:874714 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-11-01 20:16:30,837 INFO MainThread:874714 [wandb_init.py:init():867] starting run threads in backend +2024-11-01 20:16:30,926 INFO MainThread:874714 [wandb_run.py:_console_start():2463] atexit reg +2024-11-01 20:16:30,926 INFO MainThread:874714 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-11-01 20:16:30,926 INFO MainThread:874714 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-11-01 20:16:30,926 INFO MainThread:874714 [wandb_run.py:_redirect():2401] Redirects installed. +2024-11-01 20:16:30,927 INFO MainThread:874714 [wandb_init.py:init():911] run started, returning control to user process +2024-11-01 20:16:30,927 INFO MainThread:874714 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_nodeterministic', 'train_set': '10M', 'batch_size': 3, 'epoch': 6, 'seed': 0, 'lr': 5e-06} +2024-11-01 20:16:30,937 WARNING MsgRouterThr:874714 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20241101_201630-cb0m59vi/run-cb0m59vi.wandb b/wandb/run-20241101_201630-cb0m59vi/run-cb0m59vi.wandb new file mode 100644 index 0000000000000000000000000000000000000000..2f729ed5ee63785d614eda18e321d036deb8ddbe Binary files /dev/null and b/wandb/run-20241101_201630-cb0m59vi/run-cb0m59vi.wandb differ diff --git a/wandb/run-20241101_201707-vk4mc2rm/files/output.log b/wandb/run-20241101_201707-vk4mc2rm/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..fea0650fe1891e5324255895046b3a6dd61cb2c6 --- /dev/null +++ b/wandb/run-20241101_201707-vk4mc2rm/files/output.log @@ -0,0 +1,17 @@ +Loading checkpoint shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:04<00:00, 2.49s/it] +tokenized_valid: Dataset({ + features: ['input_ids', 'attention_mask'], + num_rows: 600 +}) +/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +[2024-11-01 20:17:16,784] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-11-01 20:17:25,921] [INFO] [comm.py:652:init_distributed] cdb=None +[2024-11-01 20:17:25,921] [INFO] [comm.py:683:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl +Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher. +Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination +Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root... +Loading extension module cpu_adam... +Time to load cpu_adam op: 5.009636640548706 seconds +wandb: WARNING The `run_name` is currently set to the same value as `TrainingArguments.output_dir`. If this was not intended, please specify a different run name by setting the `TrainingArguments.run_name` parameter. + 0%| | 0/2739 [00:00', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'evaluation_strategy': 'steps', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False, 'use_liger_kernel': False, 'eval_use_gather_object': False} +2024-11-01 20:17:52,650 INFO MainThread:875618 [wandb_config.py:__setitem__():154] config set model/num_parameters = 3212749824 - > +2024-11-01 20:17:52,650 INFO MainThread:875618 [wandb_run.py:_config_callback():1390] config_cb model/num_parameters 3212749824 None diff --git a/wandb/run-20241101_201707-vk4mc2rm/run-vk4mc2rm.wandb b/wandb/run-20241101_201707-vk4mc2rm/run-vk4mc2rm.wandb new file mode 100644 index 0000000000000000000000000000000000000000..963b50953ad3c7b250374fb406a5e14649faa792 Binary files /dev/null and b/wandb/run-20241101_201707-vk4mc2rm/run-vk4mc2rm.wandb differ diff --git a/wandb/run-20241101_202058-hjyig8so/files/config.yaml b/wandb/run-20241101_202058-hjyig8so/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7c6db7f1d23f8df1cdf8299fcab2bac6152ba311 --- /dev/null +++ b/wandb/run-20241101_202058-hjyig8so/files/config.yaml @@ -0,0 +1,50 @@ +_wandb: + value: + cli_version: 0.18.5 + m: [] + python_version: 3.9.19 + t: + "1": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "2": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "3": + - 2 + - 13 + - 23 + - 55 + "4": 3.9.19 + "5": 0.18.5 + "6": 4.45.1 + "8": + - 5 + "12": 0.18.5 + "13": linux-x86_64 +batch_size: + value: 3 +epoch: + value: 3 +lr: + value: 5e-06 +perturbation: + value: shuffle_nondeterministic +seed: + value: 0 +train_set: + value: 10M diff --git a/wandb/run-20241101_202058-hjyig8so/files/output.log b/wandb/run-20241101_202058-hjyig8so/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..8bececc56bdd2218239b5fe586f9886d818e3a7b --- /dev/null +++ b/wandb/run-20241101_202058-hjyig8so/files/output.log @@ -0,0 +1,13 @@ +Loading checkpoint shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:05<00:00, 2.68s/it] +tokenized_valid: Dataset({ + features: ['input_ids', 'attention_mask'], + num_rows: 600 +}) +/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +[2024-11-01 20:21:06,073] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-11-01 20:21:16,315] [INFO] [comm.py:652:init_distributed] cdb=None +Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination +Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root... +Loading extension module cpu_adam... +Time to load cpu_adam op: 5.979325771331787 seconds diff --git a/wandb/run-20241101_202058-hjyig8so/files/wandb-metadata.json b/wandb/run-20241101_202058-hjyig8so/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..88f1e2d87737a298947580502e278574f060a856 --- /dev/null +++ b/wandb/run-20241101_202058-hjyig8so/files/wandb-metadata.json @@ -0,0 +1,97 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-11-02T00:20:58.552052Z", + "args": [ + "--perturbation", + "shuffle_nondeterministic", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "3", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", + "codePath": "train/train_deep_wandb.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_deep_wandb.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA RTX A6000", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888559353856", + "used": "1754804830208" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + } + ], + "cudaVersion": "11.8" +} \ No newline at end of file diff --git a/wandb/run-20241101_202058-hjyig8so/files/wandb-summary.json b/wandb/run-20241101_202058-hjyig8so/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..af1e179a37e8d33e37744fa77662147d887d724d --- /dev/null +++ b/wandb/run-20241101_202058-hjyig8so/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":22437}} \ No newline at end of file diff --git a/wandb/run-20241101_202058-hjyig8so/logs/debug-internal.log b/wandb/run-20241101_202058-hjyig8so/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..87fb8a3231bc6df1378784d2d3c04468fe03cf6a --- /dev/null +++ b/wandb/run-20241101_202058-hjyig8so/logs/debug-internal.log @@ -0,0 +1,17 @@ +{"time":"2024-11-01T20:20:58.554462889-04:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-11-01T20:20:58.554477579-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_202058-hjyig8so/logs/debug-core.log"} +{"time":"2024-11-01T20:20:58.662792703-04:00","level":"INFO","msg":"created new stream","id":"hjyig8so"} +{"time":"2024-11-01T20:20:58.662856133-04:00","level":"INFO","msg":"stream: started","id":"hjyig8so"} +{"time":"2024-11-01T20:20:58.662939214-04:00","level":"INFO","msg":"sender: started","stream_id":"hjyig8so"} +{"time":"2024-11-01T20:20:58.662917124-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"hjyig8so"}} +{"time":"2024-11-01T20:20:58.662906234-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"hjyig8so"}} +{"time":"2024-11-01T20:20:58.842115613-04:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-11-02T02:34:55.962072852-04:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2024-11-02T02:34:56.017593965-04:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2024-11-02T02:34:56.520048598-04:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2024-11-02T02:34:56.650597746-04:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2024-11-02T02:34:57.666369782-04:00","level":"INFO","msg":"stream: closing","id":"hjyig8so"} +{"time":"2024-11-02T02:34:57.666403122-04:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"hjyig8so"}} +{"time":"2024-11-02T02:34:57.666437373-04:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"hjyig8so"}} +{"time":"2024-11-02T02:34:57.666488453-04:00","level":"INFO","msg":"sender: closed","stream_id":"hjyig8so"} +{"time":"2024-11-02T02:34:57.666568473-04:00","level":"INFO","msg":"stream: closed","id":"hjyig8so"} diff --git a/wandb/run-20241101_202058-hjyig8so/logs/debug.log b/wandb/run-20241101_202058-hjyig8so/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..079314f8c9925c22a3f75e918688ab448dc91737 --- /dev/null +++ b/wandb/run-20241101_202058-hjyig8so/logs/debug.log @@ -0,0 +1,33 @@ +2024-11-01 20:20:58,550 INFO MainThread:880247 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-11-01 20:20:58,550 INFO MainThread:880247 [wandb_setup.py:_flush():79] Configure stats pid to 880247 +2024-11-01 20:20:58,550 INFO MainThread:880247 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-11-01 20:20:58,550 INFO MainThread:880247 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-11-01 20:20:58,550 INFO MainThread:880247 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-01 20:20:58,550 INFO MainThread:880247 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-01 20:20:58,550 INFO MainThread:880247 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'} +2024-11-01 20:20:58,550 INFO MainThread:880247 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-01 20:20:58,550 INFO MainThread:880247 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_202058-hjyig8so/logs/debug.log +2024-11-01 20:20:58,550 INFO MainThread:880247 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_202058-hjyig8so/logs/debug-internal.log +2024-11-01 20:20:58,550 INFO MainThread:880247 [wandb_init.py:init():621] calling init triggers +2024-11-01 20:20:58,550 INFO MainThread:880247 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-11-01 20:20:58,550 INFO MainThread:880247 [wandb_init.py:init():671] starting backend +2024-11-01 20:20:58,550 INFO MainThread:880247 [wandb_init.py:init():675] sending inform_init request +2024-11-01 20:20:58,551 INFO MainThread:880247 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-01 20:20:58,551 INFO MainThread:880247 [wandb_init.py:init():688] backend started and connected +2024-11-01 20:20:58,554 INFO MainThread:880247 [wandb_init.py:init():783] updated telemetry +2024-11-01 20:20:58,580 INFO MainThread:880247 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-11-01 20:20:58,839 INFO MainThread:880247 [wandb_init.py:init():867] starting run threads in backend +2024-11-01 20:20:58,940 INFO MainThread:880247 [wandb_run.py:_console_start():2463] atexit reg +2024-11-01 20:20:58,940 INFO MainThread:880247 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-11-01 20:20:58,940 INFO MainThread:880247 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-11-01 20:20:58,940 INFO MainThread:880247 [wandb_run.py:_redirect():2401] Redirects installed. +2024-11-01 20:20:58,942 INFO MainThread:880247 [wandb_init.py:init():911] run started, returning control to user process +2024-11-01 20:20:58,943 INFO MainThread:880247 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_nondeterministic', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0, 'lr': 5e-06} +2024-11-02 02:34:55,664 INFO MainThread:880247 [wandb_run.py:_finish():2158] finishing run yaning1001-dartmouth-college/impossible_llm_reverse/hjyig8so +2024-11-02 02:34:55,779 INFO MainThread:880247 [wandb_run.py:_atexit_cleanup():2426] got exitcode: 0 +2024-11-02 02:34:55,911 INFO MainThread:880247 [wandb_run.py:_restore():2408] restore +2024-11-02 02:34:55,912 INFO MainThread:880247 [wandb_run.py:_restore():2414] restore done +2024-11-02 02:34:57,654 INFO MainThread:880247 [wandb_run.py:_footer_history_summary_info():3975] rendering history +2024-11-02 02:34:57,655 INFO MainThread:880247 [wandb_run.py:_footer_history_summary_info():4007] rendering summary +2024-11-02 02:34:57,665 INFO MainThread:880247 [wandb_run.py:_footer_sync_info():3934] logging synced files diff --git a/wandb/run-20241105_160217-9c998ow9/files/config.yaml b/wandb/run-20241105_160217-9c998ow9/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba0e0eb5aa81d7186d91b3c8f342ad5574a4c100 --- /dev/null +++ b/wandb/run-20241105_160217-9c998ow9/files/config.yaml @@ -0,0 +1,49 @@ +_wandb: + value: + cli_version: 0.18.5 + m: [] + python_version: 3.9.19 + t: + "1": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "2": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "3": + - 13 + - 23 + - 55 + "4": 3.9.19 + "5": 0.18.5 + "6": 4.45.1 + "8": + - 5 + "12": 0.18.5 + "13": linux-x86_64 +batch_size: + value: 3 +epoch: + value: 3 +lr: + value: 5e-06 +perturbation: + value: shuffle_deterministic21 +seed: + value: 0 +train_set: + value: 10M diff --git a/wandb/run-20241105_160217-9c998ow9/files/output.log b/wandb/run-20241105_160217-9c998ow9/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..4889313e9210ddc0bf0e53a0b8deb84b34f76597 --- /dev/null +++ b/wandb/run-20241105_160217-9c998ow9/files/output.log @@ -0,0 +1,19 @@ +Traceback (most recent call last): + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/pathlib.py", line 1323, in mkdir + self._accessor.mkdir(self, mode) +FileNotFoundError: [Errno 2] No such file or directory: '/home/chunhui/.cache/huggingface/datasets/babylm_dataset_test/babylm_shuffle_deterministic21_10M_seed0/0.0.0' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 165, in + dataset = load_dataset('babylm_dataset_test.py', name=dataset_name, trust_remote_code=True) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/load.py", line 2096, in load_dataset + builder_instance.download_and_prepare( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/builder.py", line 855, in download_and_prepare + Path(self._output_dir).parent.mkdir(parents=True, exist_ok=True) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/pathlib.py", line 1327, in mkdir + self.parent.mkdir(parents=True, exist_ok=True) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/pathlib.py", line 1323, in mkdir + self._accessor.mkdir(self, mode) +OSError: [Errno 28] No space left on device: '/home/chunhui/.cache/huggingface/datasets/babylm_dataset_test/babylm_shuffle_deterministic21_10M_seed0' diff --git a/wandb/run-20241105_160217-9c998ow9/files/requirements.txt b/wandb/run-20241105_160217-9c998ow9/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..95a931302e269cc9e4fa5b719b6511f176ee2416 --- /dev/null +++ b/wandb/run-20241105_160217-9c998ow9/files/requirements.txt @@ -0,0 +1,147 @@ +funcsigs==1.0.2 +sentry-sdk==2.17.0 +multiprocess==0.70.16 +numpy==1.26.2 +pluralizer==1.2.0 +debugpy==1.6.7 +nvidia-cudnn-cu11==8.5.0.96 +deepspeed==0.15.2 +data==0.4 +pandas==2.1.3 +tomli==2.0.1 +charset-normalizer==3.3.2 +attrs==24.2.0 +aiosignal==1.3.1 +fsspec==2023.10.0 +nvidia-cusparse-cu11==11.7.4.91 +zipp==3.12.0 +mypy-extensions==1.0.0 +datasets==3.0.1 +joblib==1.3.2 +hjson==3.1.0 +traitlets==5.7.1 +stack-data==0.6.0 +transformers==4.45.1 +sympy==1.11.1 +Pygments==2.15.0 +docker-pycreds==0.4.0 +dill==0.3.8 +wheel==0.44.0 +prompt-toolkit==3.0.30 +parso==0.8.3 +ipykernel==6.23.1 +pyarrow==17.0.0 +certifi==2023.11.17 +nvidia-cufft-cu11==10.9.0.58 +six==1.16.0 +pydantic==2.9.2 +click==8.1.7 +nest-asyncio==1.5.6 +gmpy2==2.1.0 +matplotlib==3.8.2 +scipy==1.11.4 +typing_extensions==4.12.2 +statsmodels==0.14.0 +huggingface-hub==0.25.0 +frozenlist==1.4.1 +gpustat==1.1.1 +nvidia-nvtx-cu11==11.7.91 +safetensors==0.4.5 +stanza==1.9.2 +decorator==5.1.1 +seaborn==0.13.0 +sentencepiece==0.2.0 +PyYAML==6.0.1 +black==24.8.0 +protobuf==4.25.1 +pickleshare==0.7.5 +peft==0.13.0 +triton==2.0.0 +nvidia-cuda-runtime-cu11==11.7.99 +Jinja2==3.1.2 +nvidia-cusolver-cu11==11.4.0.1 +executing==1.2.0 +jupyter_client==8.1.0 +pluggy==1.3.0 +cmake==3.30.3 +pytz==2023.3.post1 +aiohappyeyeballs==2.4.2 +kiwisolver==1.4.5 +py-cpuinfo==9.0.0 +Pillow==10.1.0 +ptyprocess==0.7.0 +importlib_resources==6.4.5 +GitPython==3.1.43 +importlib-metadata==6.0.0 +iniconfig==2.0.0 +scikit-learn==1.3.2 +exceptiongroup==1.1.0 +networkx==2.8.6 +accelerate==1.0.0 +nltk==3.8.1 +shutilwhich==1.1.0 +fonttools==4.45.1 +future==0.18.3 +aiohttp==3.10.6 +wcwidth==0.2.5 +idna==3.6 +filelock==3.12.2 +pathspec==0.12.1 +jupyter_core==5.1.0 +lit==18.1.8 +nvidia-curand-cu11==10.2.10.91 +nvidia-cublas-cu11==11.10.3.66 +nvidia-ml-py==12.560.30 +msgpack==1.1.0 +python-dateutil==2.8.2 +blessed==1.20.0 +packaging==23.0 +gitdb==4.0.11 +yarl==1.13.0 +emoji==2.8.0 +tzdata==2023.3 +cycler==0.12.1 +tornado==6.2 +backcall==0.2.0 +plotnine==0.12.4 +ninja==1.11.1.1 +latex==0.7.0 +wandb==0.18.5 +setproctitle==1.3.3 +threadpoolctl==3.2.0 +requests==2.32.3 +pyparsing==3.1.1 +smmap==5.0.1 +pyzmq==23.0.0 +async-timeout==4.0.3 +annotated-types==0.7.0 +matplotlib-inline==0.1.6 +latexcodec==1.0.0 +ipython==8.0.0 +patsy==0.5.3 +contourpy==1.2.0 +multidict==6.1.0 +mizani==0.9.3 +urllib3==2.1.0 +tokenizers==0.20.0 +MarkupSafe==2.1.2 +pip==24.2 +pexpect==4.8.0 +tqdm==4.66.5 +jedi==0.18.2 +pydantic_core==2.23.4 +tempdir==0.7.1 +mpmath==1.2.1 +setuptools==72.1.0 +pytest==7.4.3 +pure-eval==0.2.2 +psutil==5.9.1 +comm==0.1.2 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +regex==2023.10.3 +platformdirs==2.5.2 +asttokens==2.2.1 +torch==2.0.0 +nvidia-nccl-cu11==2.14.3 +xxhash==3.5.0 diff --git a/wandb/run-20241105_160217-9c998ow9/files/wandb-metadata.json b/wandb/run-20241105_160217-9c998ow9/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..b390c753b166c81f0c7ecf4ce67d178566021b46 --- /dev/null +++ b/wandb/run-20241105_160217-9c998ow9/files/wandb-metadata.json @@ -0,0 +1,44 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-11-05T21:02:17.657907Z", + "args": [ + "--perturbation", + "shuffle_deterministic21", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "3", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", + "codePath": "train/train_deep_wandb.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_deep_wandb.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "disk": { + "/": { + "total": "1888559353856", + "used": "1792550322176" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + } +} \ No newline at end of file diff --git a/wandb/run-20241105_160217-9c998ow9/files/wandb-summary.json b/wandb/run-20241105_160217-9c998ow9/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..a59211b910c7b68e6827eb6c887d30d98244727c --- /dev/null +++ b/wandb/run-20241105_160217-9c998ow9/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":5}} \ No newline at end of file diff --git a/wandb/run-20241105_160217-9c998ow9/logs/debug-internal.log b/wandb/run-20241105_160217-9c998ow9/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..d93ddff5a25937661c79ebf2e4d5a47347e90255 --- /dev/null +++ b/wandb/run-20241105_160217-9c998ow9/logs/debug-internal.log @@ -0,0 +1,17 @@ +{"time":"2024-11-05T16:02:17.660055888-05:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-11-05T16:02:17.660066728-05:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_160217-9c998ow9/logs/debug-core.log"} +{"time":"2024-11-05T16:02:22.687855482-05:00","level":"INFO","msg":"created new stream","id":"9c998ow9"} +{"time":"2024-11-05T16:02:22.687889402-05:00","level":"INFO","msg":"stream: started","id":"9c998ow9"} +{"time":"2024-11-05T16:02:22.687920922-05:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"9c998ow9"}} +{"time":"2024-11-05T16:02:22.687996132-05:00","level":"INFO","msg":"sender: started","stream_id":"9c998ow9"} +{"time":"2024-11-05T16:02:22.688786716-05:00","level":"INFO","msg":"handler: started","stream_id":{"value":"9c998ow9"}} +{"time":"2024-11-05T16:02:22.860781239-05:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-11-05T16:02:23.038526868-05:00","level":"INFO","msg":"stream: closing","id":"9c998ow9"} +{"time":"2024-11-05T16:02:23.038573968-05:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2024-11-05T16:02:23.038681169-05:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2024-11-05T16:02:23.211207106-05:00","level":"ERROR","msg":"sender: sendDefer: failed to build job artifact","error":"failed to write data to file: write /tmp/tmpfile-4264971611: no space left on device"} +{"time":"2024-11-05T16:02:23.508900085-05:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2024-11-05T16:02:23.62308555-05:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"9c998ow9"}} +{"time":"2024-11-05T16:02:23.62314132-05:00","level":"INFO","msg":"sender: closed","stream_id":"9c998ow9"} +{"time":"2024-11-05T16:02:23.62312676-05:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"9c998ow9"}} +{"time":"2024-11-05T16:02:23.62319461-05:00","level":"INFO","msg":"stream: closed","id":"9c998ow9"} diff --git a/wandb/run-20241105_160217-9c998ow9/logs/debug.log b/wandb/run-20241105_160217-9c998ow9/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..7978d8994ca3881c412a78dfd5c9e4d5bab98e3a --- /dev/null +++ b/wandb/run-20241105_160217-9c998ow9/logs/debug.log @@ -0,0 +1,27 @@ +2024-11-05 16:02:17,654 INFO MainThread:1770492 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-11-05 16:02:17,654 INFO MainThread:1770492 [wandb_setup.py:_flush():79] Configure stats pid to 1770492 +2024-11-05 16:02:17,654 INFO MainThread:1770492 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-11-05 16:02:17,654 INFO MainThread:1770492 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-11-05 16:02:17,654 INFO MainThread:1770492 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-05 16:02:17,654 INFO MainThread:1770492 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-05 16:02:17,655 INFO MainThread:1770492 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'} +2024-11-05 16:02:17,655 INFO MainThread:1770492 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-05 16:02:17,655 INFO MainThread:1770492 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_160217-9c998ow9/logs/debug.log +2024-11-05 16:02:17,655 INFO MainThread:1770492 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_160217-9c998ow9/logs/debug-internal.log +2024-11-05 16:02:17,655 INFO MainThread:1770492 [wandb_init.py:init():621] calling init triggers +2024-11-05 16:02:17,655 INFO MainThread:1770492 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-11-05 16:02:17,655 INFO MainThread:1770492 [wandb_init.py:init():671] starting backend +2024-11-05 16:02:17,655 INFO MainThread:1770492 [wandb_init.py:init():675] sending inform_init request +2024-11-05 16:02:17,657 INFO MainThread:1770492 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-05 16:02:17,657 INFO MainThread:1770492 [wandb_init.py:init():688] backend started and connected +2024-11-05 16:02:17,662 INFO MainThread:1770492 [wandb_init.py:init():783] updated telemetry +2024-11-05 16:02:17,680 INFO MainThread:1770492 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-11-05 16:02:22,856 INFO MainThread:1770492 [wandb_init.py:init():867] starting run threads in backend +2024-11-05 16:02:22,958 INFO MainThread:1770492 [wandb_run.py:_console_start():2463] atexit reg +2024-11-05 16:02:22,959 INFO MainThread:1770492 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-11-05 16:02:22,959 INFO MainThread:1770492 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-11-05 16:02:22,959 INFO MainThread:1770492 [wandb_run.py:_redirect():2401] Redirects installed. +2024-11-05 16:02:22,960 INFO MainThread:1770492 [wandb_init.py:init():911] run started, returning control to user process +2024-11-05 16:02:22,960 INFO MainThread:1770492 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_deterministic21', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0, 'lr': 5e-06} +2024-11-05 16:02:23,038 WARNING MsgRouterThr:1770492 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20241105_160217-9c998ow9/run-9c998ow9.wandb b/wandb/run-20241105_160217-9c998ow9/run-9c998ow9.wandb new file mode 100644 index 0000000000000000000000000000000000000000..22b2a44cacbc03876cbaaa1cdd94da7913d227d4 Binary files /dev/null and b/wandb/run-20241105_160217-9c998ow9/run-9c998ow9.wandb differ diff --git a/wandb/run-20241105_161832-c18fx9uc/run-c18fx9uc.wandb b/wandb/run-20241105_161832-c18fx9uc/run-c18fx9uc.wandb new file mode 100644 index 0000000000000000000000000000000000000000..53f7752016245227a1107ef232785eac9d108476 --- /dev/null +++ b/wandb/run-20241105_161832-c18fx9uc/run-c18fx9uc.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4d074e835c8d6252b36483449fe7a97638b0a3adb60b47e649615f26f36071c +size 294912 diff --git a/wandb/run-20241105_161832-sl992h9i/run-sl992h9i.wandb b/wandb/run-20241105_161832-sl992h9i/run-sl992h9i.wandb new file mode 100644 index 0000000000000000000000000000000000000000..3179f9f3c93ffda651f230cd9a9328cc13aa8f9a --- /dev/null +++ b/wandb/run-20241105_161832-sl992h9i/run-sl992h9i.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f501342874c99c84fbe32f75958bfbae3e08d3fde9e37cddf00e9daed94a2f3 +size 262144 diff --git a/wandb/run-20241106_224416-lo18g7x2/files/config.yaml b/wandb/run-20241106_224416-lo18g7x2/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..507bc548a43d1c7afb495d052831f04f348b9c17 --- /dev/null +++ b/wandb/run-20241106_224416-lo18g7x2/files/config.yaml @@ -0,0 +1,49 @@ +_wandb: + value: + cli_version: 0.18.5 + m: [] + python_version: 3.9.19 + t: + "1": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "2": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "3": + - 13 + - 23 + - 55 + "4": 3.9.19 + "5": 0.18.5 + "6": 4.45.1 + "8": + - 5 + "12": 0.18.5 + "13": linux-x86_64 +batch_size: + value: 3 +epoch: + value: 3 +lr: + value: 5e-06 +perturbation: + value: shuffle_deterministic84 +seed: + value: 0 +train_set: + value: 10M diff --git a/wandb/run-20241106_224416-lo18g7x2/files/output.log b/wandb/run-20241106_224416-lo18g7x2/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..02564b9bbad7ffd424b385d3e3f1457cf6d86b76 --- /dev/null +++ b/wandb/run-20241106_224416-lo18g7x2/files/output.log @@ -0,0 +1,84 @@ +config.json: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 844/844 [00:00<00:00, 382kB/s] +model.safetensors.index.json: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 20.9k/20.9k [00:00<00:00, 23.6MB/s] +Downloading shards: 0%| | 0/2 [00:01 + model = AutoModelForCausalLM.from_pretrained(model_name, + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/models/auto/auto_factory.py", line 564, in from_pretrained + return model_class.from_pretrained( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/modeling_utils.py", line 3769, in from_pretrained + resolved_archive_file, sharded_metadata = get_checkpoint_shard_files( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/utils/hub.py", line 1098, in get_checkpoint_shard_files + cached_filename = cached_file( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/utils/hub.py", line 403, in cached_file + resolved_file = hf_hub_download( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/utils/_deprecation.py", line 101, in inner_f + return f(*args, **kwargs) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn + return fn(*args, **kwargs) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1232, in hf_hub_download + return _hf_hub_download_to_cache_dir( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1381, in _hf_hub_download_to_cache_dir + _download_to_tmp_and_move( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1915, in _download_to_tmp_and_move + http_get( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 558, in http_get + return http_get( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/tqdm/std.py", line 1140, in __exit__ + self.close() + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/tqdm/std.py", line 1290, in close + fp_write('') + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/tqdm/std.py", line 1287, in fp_write + self.fp.write(str(s)) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/tqdm/utils.py", line 196, in inner + return func(*args, **kwargs) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/wandb/sdk/lib/redirect.py", line 648, in write + cb(data) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/wandb/sdk/wandb_run.py", line 2386, in + lambda data: self._console_raw_callback("stderr", data), + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/wandb/sdk/wandb_run.py", line 403, in wrapper_fn + return func(self, *args, **kwargs) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/wandb/sdk/wandb_run.py", line 1547, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/wandb/sdk/interface/interface.py", line 721, in publish_output_raw + self._publish_output_raw(o) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/wandb/sdk/interface/interface_shared.py", line 79, in _publish_output_raw + self._publish(rec) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/wandb/sdk/interface/interface_sock.py", line 51, in _publish + self._sock_client.send_record_publish(record) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/wandb/sdk/lib/sock_client.py", line 225, in send_record_publish + self.send_server_request(server_req) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/wandb/sdk/lib/sock_client.py", line 157, in send_server_request + self._send_message(msg) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/wandb/sdk/lib/sock_client.py", line 154, in _send_message + self._sendall_with_error_handle(header + data) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/wandb/sdk/lib/sock_client.py", line 132, in _sendall_with_error_handle + sent = self._sock.send(data) +KeyboardInterrupt diff --git a/wandb/run-20241106_224416-lo18g7x2/files/requirements.txt b/wandb/run-20241106_224416-lo18g7x2/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..95a931302e269cc9e4fa5b719b6511f176ee2416 --- /dev/null +++ b/wandb/run-20241106_224416-lo18g7x2/files/requirements.txt @@ -0,0 +1,147 @@ +funcsigs==1.0.2 +sentry-sdk==2.17.0 +multiprocess==0.70.16 +numpy==1.26.2 +pluralizer==1.2.0 +debugpy==1.6.7 +nvidia-cudnn-cu11==8.5.0.96 +deepspeed==0.15.2 +data==0.4 +pandas==2.1.3 +tomli==2.0.1 +charset-normalizer==3.3.2 +attrs==24.2.0 +aiosignal==1.3.1 +fsspec==2023.10.0 +nvidia-cusparse-cu11==11.7.4.91 +zipp==3.12.0 +mypy-extensions==1.0.0 +datasets==3.0.1 +joblib==1.3.2 +hjson==3.1.0 +traitlets==5.7.1 +stack-data==0.6.0 +transformers==4.45.1 +sympy==1.11.1 +Pygments==2.15.0 +docker-pycreds==0.4.0 +dill==0.3.8 +wheel==0.44.0 +prompt-toolkit==3.0.30 +parso==0.8.3 +ipykernel==6.23.1 +pyarrow==17.0.0 +certifi==2023.11.17 +nvidia-cufft-cu11==10.9.0.58 +six==1.16.0 +pydantic==2.9.2 +click==8.1.7 +nest-asyncio==1.5.6 +gmpy2==2.1.0 +matplotlib==3.8.2 +scipy==1.11.4 +typing_extensions==4.12.2 +statsmodels==0.14.0 +huggingface-hub==0.25.0 +frozenlist==1.4.1 +gpustat==1.1.1 +nvidia-nvtx-cu11==11.7.91 +safetensors==0.4.5 +stanza==1.9.2 +decorator==5.1.1 +seaborn==0.13.0 +sentencepiece==0.2.0 +PyYAML==6.0.1 +black==24.8.0 +protobuf==4.25.1 +pickleshare==0.7.5 +peft==0.13.0 +triton==2.0.0 +nvidia-cuda-runtime-cu11==11.7.99 +Jinja2==3.1.2 +nvidia-cusolver-cu11==11.4.0.1 +executing==1.2.0 +jupyter_client==8.1.0 +pluggy==1.3.0 +cmake==3.30.3 +pytz==2023.3.post1 +aiohappyeyeballs==2.4.2 +kiwisolver==1.4.5 +py-cpuinfo==9.0.0 +Pillow==10.1.0 +ptyprocess==0.7.0 +importlib_resources==6.4.5 +GitPython==3.1.43 +importlib-metadata==6.0.0 +iniconfig==2.0.0 +scikit-learn==1.3.2 +exceptiongroup==1.1.0 +networkx==2.8.6 +accelerate==1.0.0 +nltk==3.8.1 +shutilwhich==1.1.0 +fonttools==4.45.1 +future==0.18.3 +aiohttp==3.10.6 +wcwidth==0.2.5 +idna==3.6 +filelock==3.12.2 +pathspec==0.12.1 +jupyter_core==5.1.0 +lit==18.1.8 +nvidia-curand-cu11==10.2.10.91 +nvidia-cublas-cu11==11.10.3.66 +nvidia-ml-py==12.560.30 +msgpack==1.1.0 +python-dateutil==2.8.2 +blessed==1.20.0 +packaging==23.0 +gitdb==4.0.11 +yarl==1.13.0 +emoji==2.8.0 +tzdata==2023.3 +cycler==0.12.1 +tornado==6.2 +backcall==0.2.0 +plotnine==0.12.4 +ninja==1.11.1.1 +latex==0.7.0 +wandb==0.18.5 +setproctitle==1.3.3 +threadpoolctl==3.2.0 +requests==2.32.3 +pyparsing==3.1.1 +smmap==5.0.1 +pyzmq==23.0.0 +async-timeout==4.0.3 +annotated-types==0.7.0 +matplotlib-inline==0.1.6 +latexcodec==1.0.0 +ipython==8.0.0 +patsy==0.5.3 +contourpy==1.2.0 +multidict==6.1.0 +mizani==0.9.3 +urllib3==2.1.0 +tokenizers==0.20.0 +MarkupSafe==2.1.2 +pip==24.2 +pexpect==4.8.0 +tqdm==4.66.5 +jedi==0.18.2 +pydantic_core==2.23.4 +tempdir==0.7.1 +mpmath==1.2.1 +setuptools==72.1.0 +pytest==7.4.3 +pure-eval==0.2.2 +psutil==5.9.1 +comm==0.1.2 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +regex==2023.10.3 +platformdirs==2.5.2 +asttokens==2.2.1 +torch==2.0.0 +nvidia-nccl-cu11==2.14.3 +xxhash==3.5.0 diff --git a/wandb/run-20241106_224416-lo18g7x2/files/wandb-metadata.json b/wandb/run-20241106_224416-lo18g7x2/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..d7d1c112eb67bf8108ee4a6951c6645d7108fe81 --- /dev/null +++ b/wandb/run-20241106_224416-lo18g7x2/files/wandb-metadata.json @@ -0,0 +1,97 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-11-07T03:44:16.698776Z", + "args": [ + "--perturbation", + "shuffle_deterministic84", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "3", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", + "codePath": "train/train_deep_wandb.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_deep_wandb.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA RTX A6000", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888559353856", + "used": "1774852624384" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + } + ], + "cudaVersion": "11.8" +} \ No newline at end of file diff --git a/wandb/run-20241106_224416-lo18g7x2/files/wandb-summary.json b/wandb/run-20241106_224416-lo18g7x2/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..1d52051e315a7a21a9d9e5a40a517408bb086162 --- /dev/null +++ b/wandb/run-20241106_224416-lo18g7x2/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":2}} \ No newline at end of file diff --git a/wandb/run-20241106_224416-lo18g7x2/logs/debug-internal.log b/wandb/run-20241106_224416-lo18g7x2/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..623743984ad90fbacacf4c3c2b9561012407a208 --- /dev/null +++ b/wandb/run-20241106_224416-lo18g7x2/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2024-11-06T22:44:16.700609434-05:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-11-06T22:44:16.700619695-05:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241106_224416-lo18g7x2/logs/debug-core.log"} +{"time":"2024-11-06T22:44:16.805661452-05:00","level":"INFO","msg":"created new stream","id":"lo18g7x2"} +{"time":"2024-11-06T22:44:16.805701992-05:00","level":"INFO","msg":"stream: started","id":"lo18g7x2"} +{"time":"2024-11-06T22:44:16.805773112-05:00","level":"INFO","msg":"sender: started","stream_id":"lo18g7x2"} +{"time":"2024-11-06T22:44:16.805726032-05:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"lo18g7x2"}} +{"time":"2024-11-06T22:44:16.805745302-05:00","level":"INFO","msg":"handler: started","stream_id":{"value":"lo18g7x2"}} +{"time":"2024-11-06T22:44:16.99540646-05:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-11-06T22:44:19.096401433-05:00","level":"INFO","msg":"stream: closing","id":"lo18g7x2"} +{"time":"2024-11-06T22:44:19.096481364-05:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2024-11-06T22:44:19.097685383-05:00","level":"INFO","msg":"Stopped system monitor"} diff --git a/wandb/run-20241106_224416-lo18g7x2/logs/debug.log b/wandb/run-20241106_224416-lo18g7x2/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..423dffdc1e9518f7fe04a0b0b7417bf193b72677 --- /dev/null +++ b/wandb/run-20241106_224416-lo18g7x2/logs/debug.log @@ -0,0 +1,27 @@ +2024-11-06 22:44:16,696 INFO MainThread:1982915 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-11-06 22:44:16,696 INFO MainThread:1982915 [wandb_setup.py:_flush():79] Configure stats pid to 1982915 +2024-11-06 22:44:16,697 INFO MainThread:1982915 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-11-06 22:44:16,697 INFO MainThread:1982915 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-11-06 22:44:16,697 INFO MainThread:1982915 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-06 22:44:16,697 INFO MainThread:1982915 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-06 22:44:16,697 INFO MainThread:1982915 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'} +2024-11-06 22:44:16,697 INFO MainThread:1982915 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-06 22:44:16,697 INFO MainThread:1982915 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241106_224416-lo18g7x2/logs/debug.log +2024-11-06 22:44:16,697 INFO MainThread:1982915 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241106_224416-lo18g7x2/logs/debug-internal.log +2024-11-06 22:44:16,697 INFO MainThread:1982915 [wandb_init.py:init():621] calling init triggers +2024-11-06 22:44:16,697 INFO MainThread:1982915 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-11-06 22:44:16,697 INFO MainThread:1982915 [wandb_init.py:init():671] starting backend +2024-11-06 22:44:16,697 INFO MainThread:1982915 [wandb_init.py:init():675] sending inform_init request +2024-11-06 22:44:16,698 INFO MainThread:1982915 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-06 22:44:16,698 INFO MainThread:1982915 [wandb_init.py:init():688] backend started and connected +2024-11-06 22:44:16,701 INFO MainThread:1982915 [wandb_init.py:init():783] updated telemetry +2024-11-06 22:44:16,728 INFO MainThread:1982915 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-11-06 22:44:16,992 INFO MainThread:1982915 [wandb_init.py:init():867] starting run threads in backend +2024-11-06 22:44:17,081 INFO MainThread:1982915 [wandb_run.py:_console_start():2463] atexit reg +2024-11-06 22:44:17,081 INFO MainThread:1982915 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-11-06 22:44:17,081 INFO MainThread:1982915 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-11-06 22:44:17,081 INFO MainThread:1982915 [wandb_run.py:_redirect():2401] Redirects installed. +2024-11-06 22:44:17,083 INFO MainThread:1982915 [wandb_init.py:init():911] run started, returning control to user process +2024-11-06 22:44:17,083 INFO MainThread:1982915 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_deterministic84', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0, 'lr': 5e-06} +2024-11-06 22:44:19,096 WARNING MsgRouterThr:1982915 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20241106_224416-lo18g7x2/run-lo18g7x2.wandb b/wandb/run-20241106_224416-lo18g7x2/run-lo18g7x2.wandb new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/wandb/run-20241107_160909-8q592a50/files/wandb-summary.json b/wandb/run-20241107_160909-8q592a50/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..eeb691803bee7cfbd9d8e2adf93c92d5d5f5718b --- /dev/null +++ b/wandb/run-20241107_160909-8q592a50/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":34215}} \ No newline at end of file diff --git a/wandb/run-20241107_160909-8q592a50/logs/debug-internal.log b/wandb/run-20241107_160909-8q592a50/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..c83fea36595a557cd35c4aef28b27b5a5b9c8f0b --- /dev/null +++ b/wandb/run-20241107_160909-8q592a50/logs/debug-internal.log @@ -0,0 +1,22 @@ +{"time":"2024-11-07T16:09:09.627257557-05:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-11-07T16:09:09.627273877-05:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241107_160909-8q592a50/logs/debug-core.log"} +{"time":"2024-11-07T16:09:09.736132024-05:00","level":"INFO","msg":"created new stream","id":"8q592a50"} +{"time":"2024-11-07T16:09:09.736190415-05:00","level":"INFO","msg":"stream: started","id":"8q592a50"} +{"time":"2024-11-07T16:09:09.736261335-05:00","level":"INFO","msg":"sender: started","stream_id":"8q592a50"} +{"time":"2024-11-07T16:09:09.736256875-05:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"8q592a50"}} +{"time":"2024-11-07T16:09:09.736242225-05:00","level":"INFO","msg":"handler: started","stream_id":{"value":"8q592a50"}} +{"time":"2024-11-07T16:09:09.944026685-05:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-11-07T16:35:31.062475192-05:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"} +{"time":"2024-11-07T19:30:50.139777936-05:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"} +{"time":"2024-11-07T19:54:28.009675125-05:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"} +{"time":"2024-11-07T19:56:30.305350337-05:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"} +{"time":"2024-11-07T20:36:32.780672168-05:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/yaning1001-dartmouth-college/exp-impo-shuffle/8q592a50/file_stream"} +{"time":"2024-11-08T01:39:25.457922596-05:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2024-11-08T01:39:25.528084411-05:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2024-11-08T01:39:26.148982984-05:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2024-11-08T01:39:26.340574729-05:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2024-11-08T01:39:27.35402693-05:00","level":"INFO","msg":"stream: closing","id":"8q592a50"} +{"time":"2024-11-08T01:39:27.3540583-05:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"8q592a50"}} +{"time":"2024-11-08T01:39:27.35407699-05:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"8q592a50"}} +{"time":"2024-11-08T01:39:27.35408603-05:00","level":"INFO","msg":"sender: closed","stream_id":"8q592a50"} +{"time":"2024-11-08T01:39:27.354181931-05:00","level":"INFO","msg":"stream: closed","id":"8q592a50"} diff --git a/wandb/run-20241107_160909-8q592a50/logs/debug.log b/wandb/run-20241107_160909-8q592a50/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..97f4804258589907a8836b2b6d6273f4d9a593ca --- /dev/null +++ b/wandb/run-20241107_160909-8q592a50/logs/debug.log @@ -0,0 +1,33 @@ +2024-11-07 16:09:09,623 INFO MainThread:2097823 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-11-07 16:09:09,623 INFO MainThread:2097823 [wandb_setup.py:_flush():79] Configure stats pid to 2097823 +2024-11-07 16:09:09,623 INFO MainThread:2097823 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-11-07 16:09:09,623 INFO MainThread:2097823 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-11-07 16:09:09,623 INFO MainThread:2097823 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-07 16:09:09,623 INFO MainThread:2097823 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-07 16:09:09,623 INFO MainThread:2097823 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'} +2024-11-07 16:09:09,623 INFO MainThread:2097823 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-07 16:09:09,623 INFO MainThread:2097823 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241107_160909-8q592a50/logs/debug.log +2024-11-07 16:09:09,623 INFO MainThread:2097823 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241107_160909-8q592a50/logs/debug-internal.log +2024-11-07 16:09:09,623 INFO MainThread:2097823 [wandb_init.py:init():621] calling init triggers +2024-11-07 16:09:09,623 INFO MainThread:2097823 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-11-07 16:09:09,623 INFO MainThread:2097823 [wandb_init.py:init():671] starting backend +2024-11-07 16:09:09,623 INFO MainThread:2097823 [wandb_init.py:init():675] sending inform_init request +2024-11-07 16:09:09,624 INFO MainThread:2097823 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-07 16:09:09,625 INFO MainThread:2097823 [wandb_init.py:init():688] backend started and connected +2024-11-07 16:09:09,627 INFO MainThread:2097823 [wandb_init.py:init():783] updated telemetry +2024-11-07 16:09:09,647 INFO MainThread:2097823 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-11-07 16:09:09,941 INFO MainThread:2097823 [wandb_init.py:init():867] starting run threads in backend +2024-11-07 16:09:10,032 INFO MainThread:2097823 [wandb_run.py:_console_start():2463] atexit reg +2024-11-07 16:09:10,032 INFO MainThread:2097823 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-11-07 16:09:10,032 INFO MainThread:2097823 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-11-07 16:09:10,032 INFO MainThread:2097823 [wandb_run.py:_redirect():2401] Redirects installed. +2024-11-07 16:09:10,034 INFO MainThread:2097823 [wandb_init.py:init():911] run started, returning control to user process +2024-11-07 16:09:10,034 INFO MainThread:2097823 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_even_odd', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0, 'lr': 5e-06} +2024-11-08 01:39:25,351 INFO MainThread:2097823 [wandb_run.py:_finish():2158] finishing run yaning1001-dartmouth-college/exp-impo-shuffle/8q592a50 +2024-11-08 01:39:25,363 INFO MainThread:2097823 [wandb_run.py:_atexit_cleanup():2426] got exitcode: 0 +2024-11-08 01:39:25,363 INFO MainThread:2097823 [wandb_run.py:_restore():2408] restore +2024-11-08 01:39:25,364 INFO MainThread:2097823 [wandb_run.py:_restore():2414] restore done +2024-11-08 01:39:27,344 INFO MainThread:2097823 [wandb_run.py:_footer_history_summary_info():3975] rendering history +2024-11-08 01:39:27,344 INFO MainThread:2097823 [wandb_run.py:_footer_history_summary_info():4007] rendering summary +2024-11-08 01:39:27,353 INFO MainThread:2097823 [wandb_run.py:_footer_sync_info():3934] logging synced files diff --git a/wandb/run-20241113_180154-b24s43he/files/output.log b/wandb/run-20241113_180154-b24s43he/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..16a4e2663ed99cd6672f90b899d650ff2ea18135 --- /dev/null +++ b/wandb/run-20241113_180154-b24s43he/files/output.log @@ -0,0 +1,21 @@ +Downloading shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [02:32<00:00, 76.11s/it] +Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:06<00:00, 3.03s/it] +Map: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 17519/17519 [00:49<00:00, 354.70 examples/s] +Map: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 18140/18140 [00:50<00:00, 359.33 examples/s] +tokenized_valid: Dataset({ + features: ['input_ids', 'attention_mask'], + num_rows: 1000 +}) +/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +[2024-11-13 18:06:38,442] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-11-13 18:06:49,433] [INFO] [comm.py:652:init_distributed] cdb=None +[2024-11-13 18:06:49,434] [INFO] [comm.py:683:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl +Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher. +Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination +Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root... +Emitting ninja build file /home/chunhui/.cache/torch_extensions/py39_cu117/cpu_adam/build.ninja... +Building extension module cpu_adam... +Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +Loading extension module cpu_adam... +Time to load cpu_adam op: 12.884292364120483 seconds diff --git a/wandb/run-20241113_180154-b24s43he/files/requirements.txt b/wandb/run-20241113_180154-b24s43he/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..95a931302e269cc9e4fa5b719b6511f176ee2416 --- /dev/null +++ b/wandb/run-20241113_180154-b24s43he/files/requirements.txt @@ -0,0 +1,147 @@ +funcsigs==1.0.2 +sentry-sdk==2.17.0 +multiprocess==0.70.16 +numpy==1.26.2 +pluralizer==1.2.0 +debugpy==1.6.7 +nvidia-cudnn-cu11==8.5.0.96 +deepspeed==0.15.2 +data==0.4 +pandas==2.1.3 +tomli==2.0.1 +charset-normalizer==3.3.2 +attrs==24.2.0 +aiosignal==1.3.1 +fsspec==2023.10.0 +nvidia-cusparse-cu11==11.7.4.91 +zipp==3.12.0 +mypy-extensions==1.0.0 +datasets==3.0.1 +joblib==1.3.2 +hjson==3.1.0 +traitlets==5.7.1 +stack-data==0.6.0 +transformers==4.45.1 +sympy==1.11.1 +Pygments==2.15.0 +docker-pycreds==0.4.0 +dill==0.3.8 +wheel==0.44.0 +prompt-toolkit==3.0.30 +parso==0.8.3 +ipykernel==6.23.1 +pyarrow==17.0.0 +certifi==2023.11.17 +nvidia-cufft-cu11==10.9.0.58 +six==1.16.0 +pydantic==2.9.2 +click==8.1.7 +nest-asyncio==1.5.6 +gmpy2==2.1.0 +matplotlib==3.8.2 +scipy==1.11.4 +typing_extensions==4.12.2 +statsmodels==0.14.0 +huggingface-hub==0.25.0 +frozenlist==1.4.1 +gpustat==1.1.1 +nvidia-nvtx-cu11==11.7.91 +safetensors==0.4.5 +stanza==1.9.2 +decorator==5.1.1 +seaborn==0.13.0 +sentencepiece==0.2.0 +PyYAML==6.0.1 +black==24.8.0 +protobuf==4.25.1 +pickleshare==0.7.5 +peft==0.13.0 +triton==2.0.0 +nvidia-cuda-runtime-cu11==11.7.99 +Jinja2==3.1.2 +nvidia-cusolver-cu11==11.4.0.1 +executing==1.2.0 +jupyter_client==8.1.0 +pluggy==1.3.0 +cmake==3.30.3 +pytz==2023.3.post1 +aiohappyeyeballs==2.4.2 +kiwisolver==1.4.5 +py-cpuinfo==9.0.0 +Pillow==10.1.0 +ptyprocess==0.7.0 +importlib_resources==6.4.5 +GitPython==3.1.43 +importlib-metadata==6.0.0 +iniconfig==2.0.0 +scikit-learn==1.3.2 +exceptiongroup==1.1.0 +networkx==2.8.6 +accelerate==1.0.0 +nltk==3.8.1 +shutilwhich==1.1.0 +fonttools==4.45.1 +future==0.18.3 +aiohttp==3.10.6 +wcwidth==0.2.5 +idna==3.6 +filelock==3.12.2 +pathspec==0.12.1 +jupyter_core==5.1.0 +lit==18.1.8 +nvidia-curand-cu11==10.2.10.91 +nvidia-cublas-cu11==11.10.3.66 +nvidia-ml-py==12.560.30 +msgpack==1.1.0 +python-dateutil==2.8.2 +blessed==1.20.0 +packaging==23.0 +gitdb==4.0.11 +yarl==1.13.0 +emoji==2.8.0 +tzdata==2023.3 +cycler==0.12.1 +tornado==6.2 +backcall==0.2.0 +plotnine==0.12.4 +ninja==1.11.1.1 +latex==0.7.0 +wandb==0.18.5 +setproctitle==1.3.3 +threadpoolctl==3.2.0 +requests==2.32.3 +pyparsing==3.1.1 +smmap==5.0.1 +pyzmq==23.0.0 +async-timeout==4.0.3 +annotated-types==0.7.0 +matplotlib-inline==0.1.6 +latexcodec==1.0.0 +ipython==8.0.0 +patsy==0.5.3 +contourpy==1.2.0 +multidict==6.1.0 +mizani==0.9.3 +urllib3==2.1.0 +tokenizers==0.20.0 +MarkupSafe==2.1.2 +pip==24.2 +pexpect==4.8.0 +tqdm==4.66.5 +jedi==0.18.2 +pydantic_core==2.23.4 +tempdir==0.7.1 +mpmath==1.2.1 +setuptools==72.1.0 +pytest==7.4.3 +pure-eval==0.2.2 +psutil==5.9.1 +comm==0.1.2 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +regex==2023.10.3 +platformdirs==2.5.2 +asttokens==2.2.1 +torch==2.0.0 +nvidia-nccl-cu11==2.14.3 +xxhash==3.5.0 diff --git a/wandb/run-20241113_180154-b24s43he/files/wandb-metadata.json b/wandb/run-20241113_180154-b24s43he/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..642e313c004581a347080a7ed2b1d48f3907e8ba --- /dev/null +++ b/wandb/run-20241113_180154-b24s43he/files/wandb-metadata.json @@ -0,0 +1,97 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-11-13T23:01:54.132415Z", + "args": [ + "--perturbation", + "reverse_full", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "3", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_ftp.py", + "codePath": "train/train_ftp.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_ftp.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA RTX A6000", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888559353856", + "used": "1744802353152" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + } + ], + "cudaVersion": "11.8" +} \ No newline at end of file diff --git a/wandb/run-20241113_180154-b24s43he/logs/debug-internal.log b/wandb/run-20241113_180154-b24s43he/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..6f195bfc6c49890bf3fe6376bf6b84bd4c52f527 --- /dev/null +++ b/wandb/run-20241113_180154-b24s43he/logs/debug-internal.log @@ -0,0 +1,8 @@ +{"time":"2024-11-13T18:01:54.13860551-05:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-11-13T18:01:54.13864971-05:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241113_180154-b24s43he/logs/debug-core.log"} +{"time":"2024-11-13T18:01:54.253114604-05:00","level":"INFO","msg":"created new stream","id":"b24s43he"} +{"time":"2024-11-13T18:01:54.253170845-05:00","level":"INFO","msg":"stream: started","id":"b24s43he"} +{"time":"2024-11-13T18:01:54.253202515-05:00","level":"INFO","msg":"sender: started","stream_id":"b24s43he"} +{"time":"2024-11-13T18:01:54.253189635-05:00","level":"INFO","msg":"handler: started","stream_id":{"value":"b24s43he"}} +{"time":"2024-11-13T18:01:54.253256345-05:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"b24s43he"}} +{"time":"2024-11-13T18:01:54.566116725-05:00","level":"INFO","msg":"Starting system monitor"} diff --git a/wandb/run-20241113_180154-b24s43he/logs/debug.log b/wandb/run-20241113_180154-b24s43he/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b06e18e6f3c39132135a563e7178ebfc6123838b --- /dev/null +++ b/wandb/run-20241113_180154-b24s43he/logs/debug.log @@ -0,0 +1,26 @@ +2024-11-13 18:01:54,129 INFO MainThread:2553104 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-11-13 18:01:54,129 INFO MainThread:2553104 [wandb_setup.py:_flush():79] Configure stats pid to 2553104 +2024-11-13 18:01:54,129 INFO MainThread:2553104 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-11-13 18:01:54,129 INFO MainThread:2553104 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-11-13 18:01:54,129 INFO MainThread:2553104 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-13 18:01:54,130 INFO MainThread:2553104 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-13 18:01:54,130 INFO MainThread:2553104 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_ftp.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_ftp.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_ftp.py'} +2024-11-13 18:01:54,130 INFO MainThread:2553104 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-13 18:01:54,130 INFO MainThread:2553104 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241113_180154-b24s43he/logs/debug.log +2024-11-13 18:01:54,130 INFO MainThread:2553104 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241113_180154-b24s43he/logs/debug-internal.log +2024-11-13 18:01:54,130 INFO MainThread:2553104 [wandb_init.py:init():621] calling init triggers +2024-11-13 18:01:54,130 INFO MainThread:2553104 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-11-13 18:01:54,130 INFO MainThread:2553104 [wandb_init.py:init():671] starting backend +2024-11-13 18:01:54,130 INFO MainThread:2553104 [wandb_init.py:init():675] sending inform_init request +2024-11-13 18:01:54,131 INFO MainThread:2553104 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-13 18:01:54,132 INFO MainThread:2553104 [wandb_init.py:init():688] backend started and connected +2024-11-13 18:01:54,135 INFO MainThread:2553104 [wandb_init.py:init():783] updated telemetry +2024-11-13 18:01:54,162 INFO MainThread:2553104 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-11-13 18:01:54,563 INFO MainThread:2553104 [wandb_init.py:init():867] starting run threads in backend +2024-11-13 18:01:54,647 INFO MainThread:2553104 [wandb_run.py:_console_start():2463] atexit reg +2024-11-13 18:01:54,647 INFO MainThread:2553104 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-11-13 18:01:54,647 INFO MainThread:2553104 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-11-13 18:01:54,648 INFO MainThread:2553104 [wandb_run.py:_redirect():2401] Redirects installed. +2024-11-13 18:01:54,649 INFO MainThread:2553104 [wandb_init.py:init():911] run started, returning control to user process +2024-11-13 18:01:54,649 INFO MainThread:2553104 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_full', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0, 'lr': 5e-06} diff --git a/wandb/run-20241113_180154-y8wvn5hq/run-y8wvn5hq.wandb b/wandb/run-20241113_180154-y8wvn5hq/run-y8wvn5hq.wandb new file mode 100644 index 0000000000000000000000000000000000000000..443503be362d35181928298d5c76c347d0edab6b --- /dev/null +++ b/wandb/run-20241113_180154-y8wvn5hq/run-y8wvn5hq.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a5dbf43d30c10f4928cb268d5d5432a77f95235a23099e8d8791cffe1ed7905 +size 294912 diff --git a/wandb/run-20241129_235241-3ftwr60m/run-3ftwr60m.wandb b/wandb/run-20241129_235241-3ftwr60m/run-3ftwr60m.wandb new file mode 100644 index 0000000000000000000000000000000000000000..1efc88538182a82f9142e625c839f52a21e1e80f --- /dev/null +++ b/wandb/run-20241129_235241-3ftwr60m/run-3ftwr60m.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30bacac2a379cdea2927f35ba515203d38bd9828d05a9d0ac3feb6580dd8386e +size 327680 diff --git a/wandb/run-20241130_000210-r9oa6v8c/files/config.yaml b/wandb/run-20241130_000210-r9oa6v8c/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d07c4054eb6cea6e4e93d3fb630156b4b6a6798 --- /dev/null +++ b/wandb/run-20241130_000210-r9oa6v8c/files/config.yaml @@ -0,0 +1,50 @@ +_wandb: + value: + cli_version: 0.18.5 + m: [] + python_version: 3.9.19 + t: + "1": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "2": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "3": + - 2 + - 13 + - 23 + - 55 + "4": 3.9.19 + "5": 0.18.5 + "6": 4.45.1 + "8": + - 5 + "12": 0.18.5 + "13": linux-x86_64 +batch_size: + value: 3 +epoch: + value: 3 +lr: + value: 5e-06 +perturbation: + value: reverse_control +seed: + value: 0 +train_set: + value: 10M diff --git a/wandb/run-20241130_000210-r9oa6v8c/files/wandb-summary.json b/wandb/run-20241130_000210-r9oa6v8c/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..df7c36c738080141c121f4140c9bf38d8eb0dc3b --- /dev/null +++ b/wandb/run-20241130_000210-r9oa6v8c/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":6134}} \ No newline at end of file diff --git a/wandb/run-20241130_000210-r9oa6v8c/logs/debug-core.log b/wandb/run-20241130_000210-r9oa6v8c/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..73ada21c0b3da0279763c6964cbb72ba46f0ee11 --- /dev/null +++ b/wandb/run-20241130_000210-r9oa6v8c/logs/debug-core.log @@ -0,0 +1,48 @@ +{"time":"2024-11-30T00:02:10.246824965-05:00","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmpo_xr549q/port-3202248.txt","pid":3202248,"debug":false,"disable-analytics":false} +{"time":"2024-11-30T00:02:10.246867346-05:00","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false} +{"time":"2024-11-30T00:02:10.247704852-05:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":3202248} +{"time":"2024-11-30T00:02:10.247715422-05:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":46095,"Zone":""}} +{"time":"2024-11-30T00:02:10.264336942-05:00","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmp3dmytbm6/port-3202250.txt","pid":3202250,"debug":false,"disable-analytics":false} +{"time":"2024-11-30T00:02:10.264383852-05:00","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false} +{"time":"2024-11-30T00:02:10.264935576-05:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":3202250} +{"time":"2024-11-30T00:02:10.264942156-05:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":44257,"Zone":""}} +{"time":"2024-11-30T00:02:10.432531978-05:00","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmpkprde802/port-3202249.txt","pid":3202249,"debug":false,"disable-analytics":false} +{"time":"2024-11-30T00:02:10.432563108-05:00","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false} +{"time":"2024-11-30T00:02:10.433057102-05:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":3202249} +{"time":"2024-11-30T00:02:10.433054762-05:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":37417,"Zone":""}} +{"time":"2024-11-30T00:02:10.438319149-05:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:49092"} +{"time":"2024-11-30T00:02:10.456288079-05:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:37090"} +{"time":"2024-11-30T00:02:10.595733797-05:00","level":"INFO","msg":"handleInformInit: received","streamId":"gk41m5w8","id":"127.0.0.1:49092"} +{"time":"2024-11-30T00:02:10.625707344-05:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:38872"} +{"time":"2024-11-30T00:02:10.679885525-05:00","level":"INFO","msg":"handleInformInit: received","streamId":"inx4tpr1","id":"127.0.0.1:37090"} +{"time":"2024-11-30T00:02:10.702537709-05:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"gk41m5w8","id":"127.0.0.1:49092"} +{"time":"2024-11-30T00:02:10.781712851-05:00","level":"INFO","msg":"handleInformInit: received","streamId":"r9oa6v8c","id":"127.0.0.1:38872"} +{"time":"2024-11-30T00:02:10.786716757-05:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"inx4tpr1","id":"127.0.0.1:37090"} +{"time":"2024-11-30T00:02:10.88944601-05:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"r9oa6v8c","id":"127.0.0.1:38872"} +{"time":"2024-11-30T01:44:27.018023962-05:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"r9oa6v8c","id":"127.0.0.1:38872"} +{"time":"2024-11-30T01:44:27.018357875-05:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"r9oa6v8c","id":"127.0.0.1:38872"} +{"time":"2024-11-30T01:44:27.022116931-05:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"inx4tpr1","id":"127.0.0.1:37090"} +{"time":"2024-11-30T01:44:27.022269062-05:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"inx4tpr1","id":"127.0.0.1:37090"} +{"time":"2024-11-30T01:44:27.029060349-05:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"gk41m5w8","id":"127.0.0.1:49092"} +{"time":"2024-11-30T01:44:27.0292524-05:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"gk41m5w8","id":"127.0.0.1:49092"} +{"time":"2024-11-30T01:44:27.381653195-05:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:37090"} +{"time":"2024-11-30T01:44:27.381747286-05:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:37090"} +{"time":"2024-11-30T01:44:27.381756066-05:00","level":"INFO","msg":"server is shutting down"} +{"time":"2024-11-30T01:44:27.381771406-05:00","level":"INFO","msg":"connection: Close: initiating connection closure","id":"127.0.0.1:37090"} +{"time":"2024-11-30T01:44:27.381960397-05:00","level":"INFO","msg":"connection: Close: connection successfully closed","id":"127.0.0.1:37090"} +{"time":"2024-11-30T01:44:27.381972207-05:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:37090"} +{"time":"2024-11-30T01:44:27.381980567-05:00","level":"INFO","msg":"server is closed"} +{"time":"2024-11-30T01:44:27.390100404-05:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:38872"} +{"time":"2024-11-30T01:44:27.390133484-05:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:38872"} +{"time":"2024-11-30T01:44:27.390142444-05:00","level":"INFO","msg":"server is shutting down"} +{"time":"2024-11-30T01:44:27.390147984-05:00","level":"INFO","msg":"connection: Close: initiating connection closure","id":"127.0.0.1:38872"} +{"time":"2024-11-30T01:44:27.390225424-05:00","level":"INFO","msg":"connection: Close: connection successfully closed","id":"127.0.0.1:38872"} +{"time":"2024-11-30T01:44:27.390234705-05:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:38872"} +{"time":"2024-11-30T01:44:27.390241465-05:00","level":"INFO","msg":"server is closed"} +{"time":"2024-11-30T01:44:27.403564157-05:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:49092"} +{"time":"2024-11-30T01:44:27.403610707-05:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:49092"} +{"time":"2024-11-30T01:44:27.403618007-05:00","level":"INFO","msg":"server is shutting down"} +{"time":"2024-11-30T01:44:27.403637457-05:00","level":"INFO","msg":"connection: Close: initiating connection closure","id":"127.0.0.1:49092"} +{"time":"2024-11-30T01:44:27.403758568-05:00","level":"INFO","msg":"connection: Close: connection successfully closed","id":"127.0.0.1:49092"} +{"time":"2024-11-30T01:44:27.403787128-05:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:49092"} +{"time":"2024-11-30T01:44:27.403803228-05:00","level":"INFO","msg":"server is closed"} diff --git a/wandb/run-20241130_000210-r9oa6v8c/logs/debug.log b/wandb/run-20241130_000210-r9oa6v8c/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..ed5cd5a79d6c28597ad01b3ecaa187893c0654ab --- /dev/null +++ b/wandb/run-20241130_000210-r9oa6v8c/logs/debug.log @@ -0,0 +1,33 @@ +2024-11-30 00:02:10,777 INFO MainThread:3202249 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-11-30 00:02:10,777 INFO MainThread:3202249 [wandb_setup.py:_flush():79] Configure stats pid to 3202249 +2024-11-30 00:02:10,777 INFO MainThread:3202249 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-11-30 00:02:10,777 INFO MainThread:3202249 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-11-30 00:02:10,777 INFO MainThread:3202249 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-30 00:02:10,777 INFO MainThread:3202249 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-30 00:02:10,777 INFO MainThread:3202249 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_gpt2.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_gpt2.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_gpt2.py'} +2024-11-30 00:02:10,777 INFO MainThread:3202249 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-30 00:02:10,777 INFO MainThread:3202249 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241130_000210-r9oa6v8c/logs/debug.log +2024-11-30 00:02:10,777 INFO MainThread:3202249 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241130_000210-r9oa6v8c/logs/debug-internal.log +2024-11-30 00:02:10,778 INFO MainThread:3202249 [wandb_init.py:init():621] calling init triggers +2024-11-30 00:02:10,778 INFO MainThread:3202249 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-11-30 00:02:10,778 INFO MainThread:3202249 [wandb_init.py:init():671] starting backend +2024-11-30 00:02:10,778 INFO MainThread:3202249 [wandb_init.py:init():675] sending inform_init request +2024-11-30 00:02:10,779 INFO MainThread:3202249 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-30 00:02:10,779 INFO MainThread:3202249 [wandb_init.py:init():688] backend started and connected +2024-11-30 00:02:10,783 INFO MainThread:3202249 [wandb_init.py:init():783] updated telemetry +2024-11-30 00:02:10,808 INFO MainThread:3202249 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-11-30 00:02:11,049 INFO MainThread:3202249 [wandb_init.py:init():867] starting run threads in backend +2024-11-30 00:02:11,156 INFO MainThread:3202249 [wandb_run.py:_console_start():2463] atexit reg +2024-11-30 00:02:11,157 INFO MainThread:3202249 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-11-30 00:02:11,157 INFO MainThread:3202249 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-11-30 00:02:11,157 INFO MainThread:3202249 [wandb_run.py:_redirect():2401] Redirects installed. +2024-11-30 00:02:11,159 INFO MainThread:3202249 [wandb_init.py:init():911] run started, returning control to user process +2024-11-30 00:02:11,159 INFO MainThread:3202249 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_control', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0, 'lr': 5e-06} +2024-11-30 01:44:25,380 INFO MainThread:3202249 [wandb_run.py:_finish():2158] finishing run yaning1001-dartmouth-college/exp-impo-reverse/r9oa6v8c +2024-11-30 01:44:25,410 INFO MainThread:3202249 [wandb_run.py:_atexit_cleanup():2426] got exitcode: 0 +2024-11-30 01:44:25,410 INFO MainThread:3202249 [wandb_run.py:_restore():2408] restore +2024-11-30 01:44:25,411 INFO MainThread:3202249 [wandb_run.py:_restore():2414] restore done +2024-11-30 01:44:27,012 INFO MainThread:3202249 [wandb_run.py:_footer_history_summary_info():3975] rendering history +2024-11-30 01:44:27,013 INFO MainThread:3202249 [wandb_run.py:_footer_history_summary_info():4007] rendering summary +2024-11-30 01:44:27,017 INFO MainThread:3202249 [wandb_run.py:_footer_sync_info():3934] logging synced files