diff --git a/.gitattributes b/.gitattributes index e7ae4916c230a501b8880e133831c793ca40f999..7fdefa43d21d21942e7bb35c7b7b3458edc62f94 100644 --- a/.gitattributes +++ b/.gitattributes @@ -67,3 +67,5 @@ wandb/run-20241031_114700-3cqkhntc/run-3cqkhntc.wandb filter=lfs diff=lfs merge= wandb/run-20241030_222932-sh4rlbgu/run-sh4rlbgu.wandb filter=lfs diff=lfs merge=lfs -text wandb/run-20241113_180154-b24s43he/run-b24s43he.wandb filter=lfs diff=lfs merge=lfs -text wandb/run-20241030_233740-0fagh7s8/run-0fagh7s8.wandb filter=lfs diff=lfs merge=lfs -text +wandb/run-20241031_001055-dua2g15g/run-dua2g15g.wandb filter=lfs diff=lfs merge=lfs -text +wandb/run-20241101_200535-xloij0da/run-xloij0da.wandb filter=lfs diff=lfs merge=lfs -text diff --git a/wandb/run-20241030_112700-d8o4g13r/files/wandb-metadata.json b/wandb/run-20241030_112700-d8o4g13r/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..29766a252769f3eab109de03cfca707586b11866 --- /dev/null +++ b/wandb/run-20241030_112700-d8o4g13r/files/wandb-metadata.json @@ -0,0 +1,97 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-10-30T15:27:00.858134Z", + "args": [ + "--perturbation", + "reverse_control", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "3", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", + "codePath": "train/train_deep_wandb.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_deep_wandb.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA RTX A6000", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888559353856", + "used": "1710831083520" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + } + ], + "cudaVersion": "11.8" +} \ No newline at end of file diff --git a/wandb/run-20241030_231835-no2y29q9/files/output.log b/wandb/run-20241030_231835-no2y29q9/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..d4a2b68217c8870d3d4ced8d9ed3a1bd6de3cdc2 --- /dev/null +++ b/wandb/run-20241030_231835-no2y29q9/files/output.log @@ -0,0 +1,183 @@ +Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:05<00:00, 2.94s/it] +tokenized_valid: Dataset({ + features: ['input_ids', 'attention_mask'], + num_rows: 600 +}) +/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +[2024-10-30 23:18:43,562] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-10-30 23:18:53,300] [INFO] [comm.py:652:init_distributed] cdb=None +[2024-10-30 23:18:53,301] [INFO] [comm.py:683:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl +Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher. +Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination +Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root... +Loading extension module cpu_adam... +Time to load cpu_adam op: 5.888516664505005 seconds +wandb: WARNING The `run_name` is currently set to the same value as `TrainingArguments.output_dir`. If this was not intended, please specify a different run name by setting the `TrainingArguments.run_name` parameter. + +{'loss': 2.7525, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.0} +{'loss': 2.7605, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.0} +{'loss': 2.8008, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.0} +{'loss': 2.809, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.0} +{'loss': 2.7841, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.01} +{'loss': 2.7924, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.01} +{'loss': 2.7732, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.01} +{'loss': 2.7953, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.01} +{'loss': 2.7747, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.01} +{'loss': 2.7909, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.01} + +{'eval_loss': 2.8126132488250732, 'eval_runtime': 12.6414, 'eval_samples_per_second': 47.463, 'eval_steps_per_second': 1.028, 'epoch': 0.01} +{'loss': 2.7592, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.01} +{'loss': 2.7394, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.01} +{'loss': 2.8111, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.01} +{'loss': 2.7786, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.01} +{'loss': 2.7711, 'grad_norm': 0.0, 'learning_rate': 5e-06, 'epoch': 0.02} +{'loss': 2.7517, 'grad_norm': 6.298451900482178, 'learning_rate': 4.998288843258043e-06, 'epoch': 0.02} +{'loss': 2.7259, 'grad_norm': 6.298451900482178, 'learning_rate': 4.998288843258043e-06, 'epoch': 0.02} +{'loss': 2.7971, 'grad_norm': 6.545077323913574, 'learning_rate': 4.996577686516086e-06, 'epoch': 0.02} +{'loss': 2.6019, 'grad_norm': 5.211835861206055, 'learning_rate': 4.994866529774127e-06, 'epoch': 0.02} +{'loss': 2.6075, 'grad_norm': 3.1549763679504395, 'learning_rate': 4.99315537303217e-06, 'epoch': 0.02} +{'eval_loss': 2.605424404144287, 'eval_runtime': 11.5255, 'eval_samples_per_second': 52.058, 'eval_steps_per_second': 1.128, 'epoch': 0.02} +{'loss': 2.5804, 'grad_norm': 3.6320786476135254, 'learning_rate': 4.991444216290213e-06, 'epoch': 0.02} +{'loss': 2.5365, 'grad_norm': 2.6124260425567627, 'learning_rate': 4.989733059548255e-06, 'epoch': 0.02} +{'loss': 2.4614, 'grad_norm': 2.2906017303466797, 'learning_rate': 4.988021902806298e-06, 'epoch': 0.02} +{'loss': 2.4587, 'grad_norm': 2.029607057571411, 'learning_rate': 4.9863107460643404e-06, 'epoch': 0.02} +{'loss': 2.4088, 'grad_norm': 2.137622594833374, 'learning_rate': 4.984599589322382e-06, 'epoch': 0.03} +{'loss': 2.4206, 'grad_norm': 3.2038962841033936, 'learning_rate': 4.982888432580425e-06, 'epoch': 0.03} +{'loss': 2.3714, 'grad_norm': 2.447030782699585, 'learning_rate': 4.9811772758384674e-06, 'epoch': 0.03} +{'loss': 2.3694, 'grad_norm': 2.107184648513794, 'learning_rate': 4.97946611909651e-06, 'epoch': 0.03} +{'loss': 2.3499, 'grad_norm': 2.287409543991089, 'learning_rate': 4.977754962354553e-06, 'epoch': 0.03} +{'loss': 2.3258, 'grad_norm': 2.332838535308838, 'learning_rate': 4.976043805612594e-06, 'epoch': 0.03} +{'eval_loss': 2.3948795795440674, 'eval_runtime': 11.5106, 'eval_samples_per_second': 52.126, 'eval_steps_per_second': 1.129, 'epoch': 0.03} +{'loss': 2.2726, 'grad_norm': 1.9339438676834106, 'learning_rate': 4.974332648870637e-06, 'epoch': 0.03} +{'loss': 2.3154, 'grad_norm': 1.7203729152679443, 'learning_rate': 4.972621492128679e-06, 'epoch': 0.03} +{'loss': 2.2666, 'grad_norm': 1.9621284008026123, 'learning_rate': 4.970910335386721e-06, 'epoch': 0.03} +{'loss': 2.2531, 'grad_norm': 2.7898294925689697, 'learning_rate': 4.969199178644764e-06, 'epoch': 0.03} +{'loss': 2.2254, 'grad_norm': 1.831383466720581, 'learning_rate': 4.967488021902807e-06, 'epoch': 0.04} +{'loss': 2.1198, 'grad_norm': 1.9499363899230957, 'learning_rate': 4.965776865160849e-06, 'epoch': 0.04} +{'loss': 2.2448, 'grad_norm': 1.9152458906173706, 'learning_rate': 4.964065708418892e-06, 'epoch': 0.04} +{'loss': 2.1506, 'grad_norm': 1.651543378829956, 'learning_rate': 4.962354551676934e-06, 'epoch': 0.04} +{'loss': 2.1617, 'grad_norm': 1.5792090892791748, 'learning_rate': 4.960643394934976e-06, 'epoch': 0.04} +{'loss': 2.2038, 'grad_norm': 2.3606202602386475, 'learning_rate': 4.958932238193019e-06, 'epoch': 0.04} +{'eval_loss': 2.2738945484161377, 'eval_runtime': 11.5159, 'eval_samples_per_second': 52.102, 'eval_steps_per_second': 1.129, 'epoch': 0.04} +{'loss': 2.1666, 'grad_norm': 2.259366989135742, 'learning_rate': 4.9572210814510614e-06, 'epoch': 0.04} +{'loss': 2.0839, 'grad_norm': 1.9008065462112427, 'learning_rate': 4.955509924709104e-06, 'epoch': 0.04} +{'loss': 2.1488, 'grad_norm': 2.3995864391326904, 'learning_rate': 4.953798767967146e-06, 'epoch': 0.04} +{'loss': 2.1045, 'grad_norm': 1.932542324066162, 'learning_rate': 4.952087611225188e-06, 'epoch': 0.05} +{'loss': 2.1207, 'grad_norm': 2.3346352577209473, 'learning_rate': 4.950376454483231e-06, 'epoch': 0.05} +{'loss': 2.0903, 'grad_norm': 2.1826212406158447, 'learning_rate': 4.948665297741274e-06, 'epoch': 0.05} +{'loss': 2.1428, 'grad_norm': 2.398021936416626, 'learning_rate': 4.946954140999316e-06, 'epoch': 0.05} +{'loss': 2.0761, 'grad_norm': 2.2599124908447266, 'learning_rate': 4.945242984257359e-06, 'epoch': 0.05} +{'loss': 2.0773, 'grad_norm': 2.1450204849243164, 'learning_rate': 4.943531827515401e-06, 'epoch': 0.05} +{'loss': 2.0209, 'grad_norm': 1.935485601425171, 'learning_rate': 4.941820670773443e-06, 'epoch': 0.05} +{'eval_loss': 2.1975936889648438, 'eval_runtime': 11.5223, 'eval_samples_per_second': 52.073, 'eval_steps_per_second': 1.128, 'epoch': 0.05} +{'loss': 2.0184, 'grad_norm': 2.629650831222534, 'learning_rate': 4.940109514031486e-06, 'epoch': 0.05} +{'loss': 2.1345, 'grad_norm': 2.674574851989746, 'learning_rate': 4.9383983572895284e-06, 'epoch': 0.05} +{'loss': 2.0042, 'grad_norm': 2.356196403503418, 'learning_rate': 4.936687200547571e-06, 'epoch': 0.05} +{'loss': 2.0616, 'grad_norm': 2.331747531890869, 'learning_rate': 4.934976043805613e-06, 'epoch': 0.06} +{'loss': 2.0188, 'grad_norm': 2.0535356998443604, 'learning_rate': 4.9332648870636554e-06, 'epoch': 0.06} +{'loss': 1.9852, 'grad_norm': 2.5634617805480957, 'learning_rate': 4.931553730321697e-06, 'epoch': 0.06} +{'loss': 2.0274, 'grad_norm': 2.2701539993286133, 'learning_rate': 4.92984257357974e-06, 'epoch': 0.06} +{'loss': 1.9411, 'grad_norm': 2.1275668144226074, 'learning_rate': 4.928131416837782e-06, 'epoch': 0.06} +{'loss': 2.0064, 'grad_norm': 2.1210527420043945, 'learning_rate': 4.926420260095825e-06, 'epoch': 0.06} +{'loss': 2.0139, 'grad_norm': 2.3929598331451416, 'learning_rate': 4.924709103353868e-06, 'epoch': 0.06} +{'eval_loss': 2.145512580871582, 'eval_runtime': 11.5261, 'eval_samples_per_second': 52.056, 'eval_steps_per_second': 1.128, 'epoch': 0.06} +{'loss': 1.9654, 'grad_norm': 1.9947539567947388, 'learning_rate': 4.92299794661191e-06, 'epoch': 0.06} +{'loss': 1.9792, 'grad_norm': 2.342836618423462, 'learning_rate': 4.921286789869952e-06, 'epoch': 0.06} +{'loss': 2.039, 'grad_norm': 2.3230276107788086, 'learning_rate': 4.919575633127995e-06, 'epoch': 0.06} +{'loss': 2.0071, 'grad_norm': 1.8526898622512817, 'learning_rate': 4.917864476386037e-06, 'epoch': 0.07} +{'loss': 1.8769, 'grad_norm': 2.4270002841949463, 'learning_rate': 4.91615331964408e-06, 'epoch': 0.07} +{'loss': 1.9785, 'grad_norm': 2.003593683242798, 'learning_rate': 4.9144421629021224e-06, 'epoch': 0.07} +{'loss': 1.9254, 'grad_norm': 2.6093783378601074, 'learning_rate': 4.912731006160164e-06, 'epoch': 0.07} +{'loss': 2.0027, 'grad_norm': 2.1653261184692383, 'learning_rate': 4.911019849418207e-06, 'epoch': 0.07} +{'loss': 1.9359, 'grad_norm': 2.6015000343322754, 'learning_rate': 4.9093086926762494e-06, 'epoch': 0.07} +{'loss': 2.0521, 'grad_norm': 2.019484043121338, 'learning_rate': 4.907597535934292e-06, 'epoch': 0.07} +{'eval_loss': 2.1111998558044434, 'eval_runtime': 11.5296, 'eval_samples_per_second': 52.04, 'eval_steps_per_second': 1.128, 'epoch': 0.07} +{'loss': 1.9439, 'grad_norm': 1.9294712543487549, 'learning_rate': 4.905886379192335e-06, 'epoch': 0.07} +{'loss': 1.8964, 'grad_norm': 2.253174066543579, 'learning_rate': 4.904175222450377e-06, 'epoch': 0.07} +{'loss': 1.9529, 'grad_norm': 2.0996475219726562, 'learning_rate': 4.902464065708419e-06, 'epoch': 0.07} +{'loss': 1.9224, 'grad_norm': 2.0278995037078857, 'learning_rate': 4.900752908966462e-06, 'epoch': 0.08} +{'loss': 1.9289, 'grad_norm': 2.3247783184051514, 'learning_rate': 4.899041752224504e-06, 'epoch': 0.08} +{'loss': 1.9101, 'grad_norm': 1.927708625793457, 'learning_rate': 4.897330595482547e-06, 'epoch': 0.08} +{'loss': 1.9442, 'grad_norm': 2.276090383529663, 'learning_rate': 4.8956194387405895e-06, 'epoch': 0.08} +{'loss': 1.9425, 'grad_norm': 2.2609736919403076, 'learning_rate': 4.893908281998632e-06, 'epoch': 0.08} +{'loss': 1.8732, 'grad_norm': 2.352919101715088, 'learning_rate': 4.892197125256674e-06, 'epoch': 0.08} +{'loss': 1.9449, 'grad_norm': 2.2031309604644775, 'learning_rate': 4.8904859685147164e-06, 'epoch': 0.08} +{'eval_loss': 2.078676462173462, 'eval_runtime': 11.5386, 'eval_samples_per_second': 51.999, 'eval_steps_per_second': 1.127, 'epoch': 0.08} +{'loss': 1.8836, 'grad_norm': 2.183171272277832, 'learning_rate': 4.888774811772759e-06, 'epoch': 0.08} +{'loss': 1.919, 'grad_norm': 2.177760124206543, 'learning_rate': 4.887063655030802e-06, 'epoch': 0.08} +{'loss': 1.921, 'grad_norm': 2.321638822555542, 'learning_rate': 4.8853524982888434e-06, 'epoch': 0.09} +{'loss': 1.8998, 'grad_norm': 2.097433567047119, 'learning_rate': 4.883641341546886e-06, 'epoch': 0.09} +{'loss': 1.8696, 'grad_norm': 2.15975022315979, 'learning_rate': 4.881930184804929e-06, 'epoch': 0.09} +{'loss': 1.9312, 'grad_norm': 2.3222997188568115, 'learning_rate': 4.88021902806297e-06, 'epoch': 0.09} +{'loss': 1.8807, 'grad_norm': 2.4326765537261963, 'learning_rate': 4.878507871321013e-06, 'epoch': 0.09} +{'loss': 1.8787, 'grad_norm': 2.2705607414245605, 'learning_rate': 4.876796714579056e-06, 'epoch': 0.09} +{'loss': 1.8955, 'grad_norm': 2.7113804817199707, 'learning_rate': 4.875085557837098e-06, 'epoch': 0.09} +{'loss': 1.8414, 'grad_norm': 2.274242401123047, 'learning_rate': 4.873374401095141e-06, 'epoch': 0.09} +{'eval_loss': 2.060840606689453, 'eval_runtime': 11.5241, 'eval_samples_per_second': 52.065, 'eval_steps_per_second': 1.128, 'epoch': 0.09} +{'loss': 1.8706, 'grad_norm': 2.546048879623413, 'learning_rate': 4.8716632443531835e-06, 'epoch': 0.09} +{'loss': 1.8896, 'grad_norm': 2.1173014640808105, 'learning_rate': 4.869952087611225e-06, 'epoch': 0.09} +{'loss': 1.8787, 'grad_norm': 2.4878427982330322, 'learning_rate': 4.868240930869268e-06, 'epoch': 0.1} +{'loss': 1.8756, 'grad_norm': 2.1240787506103516, 'learning_rate': 4.8665297741273105e-06, 'epoch': 0.1} +{'loss': 1.9245, 'grad_norm': 2.1906278133392334, 'learning_rate': 4.864818617385353e-06, 'epoch': 0.1} +{'loss': 1.8033, 'grad_norm': 2.197439432144165, 'learning_rate': 4.863107460643396e-06, 'epoch': 0.1} +{'loss': 1.8828, 'grad_norm': 2.3327462673187256, 'learning_rate': 4.8613963039014374e-06, 'epoch': 0.1} +{'loss': 1.7941, 'grad_norm': 2.095132827758789, 'learning_rate': 4.85968514715948e-06, 'epoch': 0.1} +{'loss': 1.8109, 'grad_norm': 2.401387929916382, 'learning_rate': 4.857973990417523e-06, 'epoch': 0.1} +{'loss': 1.8268, 'grad_norm': 2.195265769958496, 'learning_rate': 4.856262833675565e-06, 'epoch': 0.1} +{'eval_loss': 2.0458765029907227, 'eval_runtime': 11.5321, 'eval_samples_per_second': 52.029, 'eval_steps_per_second': 1.127, 'epoch': 0.1} +{'loss': 1.8518, 'grad_norm': 2.4990336894989014, 'learning_rate': 4.854551676933608e-06, 'epoch': 0.1} +{'loss': 1.8206, 'grad_norm': 2.077829122543335, 'learning_rate': 4.8528405201916505e-06, 'epoch': 0.1} +{'loss': 1.7893, 'grad_norm': 2.1873979568481445, 'learning_rate': 4.851129363449692e-06, 'epoch': 0.11} +{'loss': 1.8129, 'grad_norm': 2.1641716957092285, 'learning_rate': 4.849418206707735e-06, 'epoch': 0.11} +{'loss': 1.881, 'grad_norm': 2.187925100326538, 'learning_rate': 4.8477070499657775e-06, 'epoch': 0.11} +{'loss': 1.863, 'grad_norm': 2.3495638370513916, 'learning_rate': 4.84599589322382e-06, 'epoch': 0.11} +{'loss': 1.8643, 'grad_norm': 2.1752283573150635, 'learning_rate': 4.844284736481863e-06, 'epoch': 0.11} +{'loss': 1.849, 'grad_norm': 2.323124647140503, 'learning_rate': 4.8425735797399045e-06, 'epoch': 0.11} +{'loss': 1.7471, 'grad_norm': 2.360914945602417, 'learning_rate': 4.840862422997947e-06, 'epoch': 0.11} +{'loss': 1.8054, 'grad_norm': 2.29608416557312, 'learning_rate': 4.839151266255989e-06, 'epoch': 0.11} +{'eval_loss': 2.017833709716797, 'eval_runtime': 11.5312, 'eval_samples_per_second': 52.033, 'eval_steps_per_second': 1.127, 'epoch': 0.11} +{'loss': 1.738, 'grad_norm': 2.081592321395874, 'learning_rate': 4.8374401095140314e-06, 'epoch': 0.11} +{'loss': 1.8248, 'grad_norm': 2.419157028198242, 'learning_rate': 4.835728952772074e-06, 'epoch': 0.11} +{'loss': 1.7872, 'grad_norm': 2.393253803253174, 'learning_rate': 4.834017796030117e-06, 'epoch': 0.12} +{'loss': 1.815, 'grad_norm': 2.2206389904022217, 'learning_rate': 4.832306639288159e-06, 'epoch': 0.12} +{'loss': 1.7481, 'grad_norm': 2.660158634185791, 'learning_rate': 4.830595482546202e-06, 'epoch': 0.12} +{'loss': 1.8719, 'grad_norm': 2.1578104496002197, 'learning_rate': 4.828884325804244e-06, 'epoch': 0.12} +{'loss': 1.8006, 'grad_norm': 2.285301685333252, 'learning_rate': 4.827173169062286e-06, 'epoch': 0.12} +{'loss': 1.8335, 'grad_norm': 2.4699130058288574, 'learning_rate': 4.825462012320329e-06, 'epoch': 0.12} +{'loss': 1.8206, 'grad_norm': 2.3087847232818604, 'learning_rate': 4.8237508555783715e-06, 'epoch': 0.12} +{'loss': 1.9022, 'grad_norm': 2.329796075820923, 'learning_rate': 4.822039698836414e-06, 'epoch': 0.12} +{'eval_loss': 2.0053842067718506, 'eval_runtime': 11.5201, 'eval_samples_per_second': 52.083, 'eval_steps_per_second': 1.128, 'epoch': 0.12} +{'loss': 1.8257, 'grad_norm': 2.540306568145752, 'learning_rate': 4.820328542094456e-06, 'epoch': 0.12} +{'loss': 1.8087, 'grad_norm': 2.6038308143615723, 'learning_rate': 4.8186173853524985e-06, 'epoch': 0.13} +{'loss': 1.7559, 'grad_norm': 2.5904085636138916, 'learning_rate': 4.816906228610541e-06, 'epoch': 0.13} +{'loss': 1.7934, 'grad_norm': 2.345602512359619, 'learning_rate': 4.815195071868584e-06, 'epoch': 0.13} +{'loss': 1.7462, 'grad_norm': 2.185978651046753, 'learning_rate': 4.813483915126626e-06, 'epoch': 0.13} +{'loss': 1.8069, 'grad_norm': 2.1208553314208984, 'learning_rate': 4.811772758384669e-06, 'epoch': 0.13} +{'loss': 1.7291, 'grad_norm': 2.478742837905884, 'learning_rate': 4.810061601642711e-06, 'epoch': 0.13} +{'loss': 1.7905, 'grad_norm': 2.2552616596221924, 'learning_rate': 4.808350444900753e-06, 'epoch': 0.13} +{'loss': 1.8371, 'grad_norm': 2.503005266189575, 'learning_rate': 4.806639288158796e-06, 'epoch': 0.13} +{'loss': 1.8357, 'grad_norm': 2.340322256088257, 'learning_rate': 4.8049281314168385e-06, 'epoch': 0.13} +{'eval_loss': 1.9985140562057495, 'eval_runtime': 11.5253, 'eval_samples_per_second': 52.059, 'eval_steps_per_second': 1.128, 'epoch': 0.13} +{'loss': 1.7605, 'grad_norm': 2.6791834831237793, 'learning_rate': 4.803216974674881e-06, 'epoch': 0.13} +{'loss': 1.8254, 'grad_norm': 2.2488231658935547, 'learning_rate': 4.801505817932923e-06, 'epoch': 0.14} +{'loss': 1.8009, 'grad_norm': 2.2643635272979736, 'learning_rate': 4.7997946611909655e-06, 'epoch': 0.14} +{'loss': 1.7361, 'grad_norm': 2.151967763900757, 'learning_rate': 4.798083504449008e-06, 'epoch': 0.14} +{'loss': 1.7842, 'grad_norm': 2.318420648574829, 'learning_rate': 4.796372347707051e-06, 'epoch': 0.14} +{'loss': 1.8062, 'grad_norm': 2.2854278087615967, 'learning_rate': 4.7946611909650925e-06, 'epoch': 0.14} +{'loss': 1.7247, 'grad_norm': 2.5026426315307617, 'learning_rate': 4.792950034223135e-06, 'epoch': 0.14} +{'loss': 1.799, 'grad_norm': 2.449467182159424, 'learning_rate': 4.791238877481178e-06, 'epoch': 0.14} +{'loss': 1.765, 'grad_norm': 2.472188949584961, 'learning_rate': 4.78952772073922e-06, 'epoch': 0.14} +{'loss': 1.7534, 'grad_norm': 2.3607966899871826, 'learning_rate': 4.787816563997262e-06, 'epoch': 0.14} +{'eval_loss': 1.986325740814209, 'eval_runtime': 11.521, 'eval_samples_per_second': 52.079, 'eval_steps_per_second': 1.128, 'epoch': 0.14} +{'loss': 1.7835, 'grad_norm': 2.830254077911377, 'learning_rate': 4.786105407255305e-06, 'epoch': 0.14} +{'loss': 1.7279, 'grad_norm': 2.241579532623291, 'learning_rate': 4.784394250513347e-06, 'epoch': 0.15} +{'loss': 1.87, 'grad_norm': 2.860384941101074, 'learning_rate': 4.78268309377139e-06, 'epoch': 0.15} +{'loss': 1.7452, 'grad_norm': 2.333618640899658, 'learning_rate': 4.7809719370294325e-06, 'epoch': 0.15} +{'loss': 1.729, 'grad_norm': 2.3267569541931152, 'learning_rate': 4.779260780287474e-06, 'epoch': 0.15} +{'loss': 1.7302, 'grad_norm': 2.508273124694824, 'learning_rate': 4.777549623545517e-06, 'epoch': 0.15} +{'loss': 1.7606, 'grad_norm': 2.3898637294769287, 'learning_rate': 4.7758384668035595e-06, 'epoch': 0.15} +{'loss': 1.7658, 'grad_norm': 2.4441757202148438, 'learning_rate': 4.774127310061602e-06, 'epoch': 0.15} +{'loss': 1.8535, 'grad_norm': 2.4708995819091797, 'learning_rate': 4.772416153319645e-06, 'epoch': 0.15} +{'loss': 1.7782, 'grad_norm': 2.721252918243408, 'learning_rate': 4.770704996577687e-06, 'epoch': 0.15} +{'eval_loss': 1.980696201324463, 'eval_runtime': 11.5248, 'eval_samples_per_second': 52.062, 'eval_steps_per_second': 1.128, 'epoch': 0.15} diff --git a/wandb/run-20241030_231835-no2y29q9/files/requirements.txt b/wandb/run-20241030_231835-no2y29q9/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..95a931302e269cc9e4fa5b719b6511f176ee2416 --- /dev/null +++ b/wandb/run-20241030_231835-no2y29q9/files/requirements.txt @@ -0,0 +1,147 @@ +funcsigs==1.0.2 +sentry-sdk==2.17.0 +multiprocess==0.70.16 +numpy==1.26.2 +pluralizer==1.2.0 +debugpy==1.6.7 +nvidia-cudnn-cu11==8.5.0.96 +deepspeed==0.15.2 +data==0.4 +pandas==2.1.3 +tomli==2.0.1 +charset-normalizer==3.3.2 +attrs==24.2.0 +aiosignal==1.3.1 +fsspec==2023.10.0 +nvidia-cusparse-cu11==11.7.4.91 +zipp==3.12.0 +mypy-extensions==1.0.0 +datasets==3.0.1 +joblib==1.3.2 +hjson==3.1.0 +traitlets==5.7.1 +stack-data==0.6.0 +transformers==4.45.1 +sympy==1.11.1 +Pygments==2.15.0 +docker-pycreds==0.4.0 +dill==0.3.8 +wheel==0.44.0 +prompt-toolkit==3.0.30 +parso==0.8.3 +ipykernel==6.23.1 +pyarrow==17.0.0 +certifi==2023.11.17 +nvidia-cufft-cu11==10.9.0.58 +six==1.16.0 +pydantic==2.9.2 +click==8.1.7 +nest-asyncio==1.5.6 +gmpy2==2.1.0 +matplotlib==3.8.2 +scipy==1.11.4 +typing_extensions==4.12.2 +statsmodels==0.14.0 +huggingface-hub==0.25.0 +frozenlist==1.4.1 +gpustat==1.1.1 +nvidia-nvtx-cu11==11.7.91 +safetensors==0.4.5 +stanza==1.9.2 +decorator==5.1.1 +seaborn==0.13.0 +sentencepiece==0.2.0 +PyYAML==6.0.1 +black==24.8.0 +protobuf==4.25.1 +pickleshare==0.7.5 +peft==0.13.0 +triton==2.0.0 +nvidia-cuda-runtime-cu11==11.7.99 +Jinja2==3.1.2 +nvidia-cusolver-cu11==11.4.0.1 +executing==1.2.0 +jupyter_client==8.1.0 +pluggy==1.3.0 +cmake==3.30.3 +pytz==2023.3.post1 +aiohappyeyeballs==2.4.2 +kiwisolver==1.4.5 +py-cpuinfo==9.0.0 +Pillow==10.1.0 +ptyprocess==0.7.0 +importlib_resources==6.4.5 +GitPython==3.1.43 +importlib-metadata==6.0.0 +iniconfig==2.0.0 +scikit-learn==1.3.2 +exceptiongroup==1.1.0 +networkx==2.8.6 +accelerate==1.0.0 +nltk==3.8.1 +shutilwhich==1.1.0 +fonttools==4.45.1 +future==0.18.3 +aiohttp==3.10.6 +wcwidth==0.2.5 +idna==3.6 +filelock==3.12.2 +pathspec==0.12.1 +jupyter_core==5.1.0 +lit==18.1.8 +nvidia-curand-cu11==10.2.10.91 +nvidia-cublas-cu11==11.10.3.66 +nvidia-ml-py==12.560.30 +msgpack==1.1.0 +python-dateutil==2.8.2 +blessed==1.20.0 +packaging==23.0 +gitdb==4.0.11 +yarl==1.13.0 +emoji==2.8.0 +tzdata==2023.3 +cycler==0.12.1 +tornado==6.2 +backcall==0.2.0 +plotnine==0.12.4 +ninja==1.11.1.1 +latex==0.7.0 +wandb==0.18.5 +setproctitle==1.3.3 +threadpoolctl==3.2.0 +requests==2.32.3 +pyparsing==3.1.1 +smmap==5.0.1 +pyzmq==23.0.0 +async-timeout==4.0.3 +annotated-types==0.7.0 +matplotlib-inline==0.1.6 +latexcodec==1.0.0 +ipython==8.0.0 +patsy==0.5.3 +contourpy==1.2.0 +multidict==6.1.0 +mizani==0.9.3 +urllib3==2.1.0 +tokenizers==0.20.0 +MarkupSafe==2.1.2 +pip==24.2 +pexpect==4.8.0 +tqdm==4.66.5 +jedi==0.18.2 +pydantic_core==2.23.4 +tempdir==0.7.1 +mpmath==1.2.1 +setuptools==72.1.0 +pytest==7.4.3 +pure-eval==0.2.2 +psutil==5.9.1 +comm==0.1.2 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +regex==2023.10.3 +platformdirs==2.5.2 +asttokens==2.2.1 +torch==2.0.0 +nvidia-nccl-cu11==2.14.3 +xxhash==3.5.0 diff --git a/wandb/run-20241030_231835-no2y29q9/files/wandb-metadata.json b/wandb/run-20241030_231835-no2y29q9/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..240cc66bf7736375c7d8af6c8c4500e0e34197b5 --- /dev/null +++ b/wandb/run-20241030_231835-no2y29q9/files/wandb-metadata.json @@ -0,0 +1,97 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-10-31T03:18:35.473254Z", + "args": [ + "--perturbation", + "reverse_full", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "3", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", + "codePath": "train/train_deep_wandb.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_deep_wandb.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA RTX A6000", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888559353856", + "used": "1711064555520" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + } + ], + "cudaVersion": "11.8" +} \ No newline at end of file diff --git a/wandb/run-20241030_231835-no2y29q9/logs/debug-internal.log b/wandb/run-20241030_231835-no2y29q9/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..33508c47662b546e1a4e612f014a1fc7cb9f27f5 --- /dev/null +++ b/wandb/run-20241030_231835-no2y29q9/logs/debug-internal.log @@ -0,0 +1,8 @@ +{"time":"2024-10-30T23:18:35.475047064-04:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-10-30T23:18:35.475059034-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_231835-no2y29q9/logs/debug-core.log"} +{"time":"2024-10-30T23:18:35.580787522-04:00","level":"INFO","msg":"created new stream","id":"no2y29q9"} +{"time":"2024-10-30T23:18:35.580820912-04:00","level":"INFO","msg":"stream: started","id":"no2y29q9"} +{"time":"2024-10-30T23:18:35.580831862-04:00","level":"INFO","msg":"sender: started","stream_id":"no2y29q9"} +{"time":"2024-10-30T23:18:35.580825052-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"no2y29q9"}} +{"time":"2024-10-30T23:18:35.580852152-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"no2y29q9"}} +{"time":"2024-10-30T23:18:35.772437179-04:00","level":"INFO","msg":"Starting system monitor"} diff --git a/wandb/run-20241030_231835-no2y29q9/logs/debug.log b/wandb/run-20241030_231835-no2y29q9/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..53e545a1534e1bfa06d7c1ef42f2b031b1afe8eb --- /dev/null +++ b/wandb/run-20241030_231835-no2y29q9/logs/debug.log @@ -0,0 +1,29 @@ +2024-10-30 23:18:35,471 INFO MainThread:457826 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-10-30 23:18:35,471 INFO MainThread:457826 [wandb_setup.py:_flush():79] Configure stats pid to 457826 +2024-10-30 23:18:35,471 INFO MainThread:457826 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-10-30 23:18:35,471 INFO MainThread:457826 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-10-30 23:18:35,471 INFO MainThread:457826 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-10-30 23:18:35,471 INFO MainThread:457826 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-10-30 23:18:35,471 INFO MainThread:457826 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'} +2024-10-30 23:18:35,471 INFO MainThread:457826 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-10-30 23:18:35,471 INFO MainThread:457826 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_231835-no2y29q9/logs/debug.log +2024-10-30 23:18:35,471 INFO MainThread:457826 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_231835-no2y29q9/logs/debug-internal.log +2024-10-30 23:18:35,471 INFO MainThread:457826 [wandb_init.py:init():621] calling init triggers +2024-10-30 23:18:35,471 INFO MainThread:457826 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-10-30 23:18:35,471 INFO MainThread:457826 [wandb_init.py:init():671] starting backend +2024-10-30 23:18:35,471 INFO MainThread:457826 [wandb_init.py:init():675] sending inform_init request +2024-10-30 23:18:35,472 INFO MainThread:457826 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-10-30 23:18:35,473 INFO MainThread:457826 [wandb_init.py:init():688] backend started and connected +2024-10-30 23:18:35,476 INFO MainThread:457826 [wandb_init.py:init():783] updated telemetry +2024-10-30 23:18:35,498 INFO MainThread:457826 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-10-30 23:18:35,769 INFO MainThread:457826 [wandb_init.py:init():867] starting run threads in backend +2024-10-30 23:18:35,880 INFO MainThread:457826 [wandb_run.py:_console_start():2463] atexit reg +2024-10-30 23:18:35,880 INFO MainThread:457826 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-10-30 23:18:35,880 INFO MainThread:457826 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-10-30 23:18:35,880 INFO MainThread:457826 [wandb_run.py:_redirect():2401] Redirects installed. +2024-10-30 23:18:35,882 INFO MainThread:457826 [wandb_init.py:init():911] run started, returning control to user process +2024-10-30 23:18:35,882 INFO MainThread:457826 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_full', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0} +2024-10-30 23:19:28,559 INFO MainThread:457826 [wandb_run.py:_config_callback():1390] config_cb None None {'vocab_size': 128256, 'max_position_embeddings': 131072, 'hidden_size': 3072, 'intermediate_size': 8192, 'num_hidden_layers': 28, 'num_attention_heads': 24, 'num_key_value_heads': 8, 'hidden_act': 'silu', 'initializer_range': 0.02, 'rms_norm_eps': 1e-05, 'pretraining_tp': 1, 'use_cache': True, 'rope_theta': 500000.0, 'rope_scaling': {'factor': 32.0, 'high_freq_factor': 4.0, 'low_freq_factor': 1.0, 'original_max_position_embeddings': 8192, 'rope_type': 'llama3'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 128, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'bfloat16', 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': True, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['LlamaForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 128000, 'pad_token_id': None, 'eos_token_id': 128001, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'meta-llama/Llama-3.2-3B', 'transformers_version': '4.45.1', 'model_type': 'llama', 'output_dir': './checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': True, 'do_predict': False, 'eval_strategy': 'steps', 'prediction_loss_only': False, 'per_device_train_batch_size': 3, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'eval_delay': 0, 'torch_empty_cache_steps': None, 'learning_rate': 5e-06, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 3, 'max_steps': -1, 'lr_scheduler_type': 'linear', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.0, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': './logs', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 1, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 150, 'save_total_limit': None, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 0, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': False, 'fp16': True, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 10, 'dataloader_num_workers': 0, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': './checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': 'deepspeed_config/train_dp_config.json', 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'evaluation_strategy': 'steps', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False, 'use_liger_kernel': False, 'eval_use_gather_object': False} +2024-10-30 23:19:28,563 INFO MainThread:457826 [wandb_config.py:__setitem__():154] config set model/num_parameters = 3212749824 - > +2024-10-30 23:19:28,563 INFO MainThread:457826 [wandb_run.py:_config_callback():1390] config_cb model/num_parameters 3212749824 None diff --git a/wandb/run-20241030_233740-a8ghkt6q/files/output.log b/wandb/run-20241030_233740-a8ghkt6q/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..aed850349a70043ac9e389e0687be70b7558dcdc --- /dev/null +++ b/wandb/run-20241030_233740-a8ghkt6q/files/output.log @@ -0,0 +1,13 @@ +Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:05<00:00, 2.95s/it] +tokenized_valid: Dataset({ + features: ['input_ids', 'attention_mask'], + num_rows: 600 +}) +/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +[2024-10-30 23:37:50,707] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-10-30 23:37:58,435] [INFO] [comm.py:652:init_distributed] cdb=None +Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination +Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root... +Loading extension module cpu_adam... +Time to load cpu_adam op: 4.681263446807861 seconds diff --git a/wandb/run-20241030_233740-a8ghkt6q/files/requirements.txt b/wandb/run-20241030_233740-a8ghkt6q/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..95a931302e269cc9e4fa5b719b6511f176ee2416 --- /dev/null +++ b/wandb/run-20241030_233740-a8ghkt6q/files/requirements.txt @@ -0,0 +1,147 @@ +funcsigs==1.0.2 +sentry-sdk==2.17.0 +multiprocess==0.70.16 +numpy==1.26.2 +pluralizer==1.2.0 +debugpy==1.6.7 +nvidia-cudnn-cu11==8.5.0.96 +deepspeed==0.15.2 +data==0.4 +pandas==2.1.3 +tomli==2.0.1 +charset-normalizer==3.3.2 +attrs==24.2.0 +aiosignal==1.3.1 +fsspec==2023.10.0 +nvidia-cusparse-cu11==11.7.4.91 +zipp==3.12.0 +mypy-extensions==1.0.0 +datasets==3.0.1 +joblib==1.3.2 +hjson==3.1.0 +traitlets==5.7.1 +stack-data==0.6.0 +transformers==4.45.1 +sympy==1.11.1 +Pygments==2.15.0 +docker-pycreds==0.4.0 +dill==0.3.8 +wheel==0.44.0 +prompt-toolkit==3.0.30 +parso==0.8.3 +ipykernel==6.23.1 +pyarrow==17.0.0 +certifi==2023.11.17 +nvidia-cufft-cu11==10.9.0.58 +six==1.16.0 +pydantic==2.9.2 +click==8.1.7 +nest-asyncio==1.5.6 +gmpy2==2.1.0 +matplotlib==3.8.2 +scipy==1.11.4 +typing_extensions==4.12.2 +statsmodels==0.14.0 +huggingface-hub==0.25.0 +frozenlist==1.4.1 +gpustat==1.1.1 +nvidia-nvtx-cu11==11.7.91 +safetensors==0.4.5 +stanza==1.9.2 +decorator==5.1.1 +seaborn==0.13.0 +sentencepiece==0.2.0 +PyYAML==6.0.1 +black==24.8.0 +protobuf==4.25.1 +pickleshare==0.7.5 +peft==0.13.0 +triton==2.0.0 +nvidia-cuda-runtime-cu11==11.7.99 +Jinja2==3.1.2 +nvidia-cusolver-cu11==11.4.0.1 +executing==1.2.0 +jupyter_client==8.1.0 +pluggy==1.3.0 +cmake==3.30.3 +pytz==2023.3.post1 +aiohappyeyeballs==2.4.2 +kiwisolver==1.4.5 +py-cpuinfo==9.0.0 +Pillow==10.1.0 +ptyprocess==0.7.0 +importlib_resources==6.4.5 +GitPython==3.1.43 +importlib-metadata==6.0.0 +iniconfig==2.0.0 +scikit-learn==1.3.2 +exceptiongroup==1.1.0 +networkx==2.8.6 +accelerate==1.0.0 +nltk==3.8.1 +shutilwhich==1.1.0 +fonttools==4.45.1 +future==0.18.3 +aiohttp==3.10.6 +wcwidth==0.2.5 +idna==3.6 +filelock==3.12.2 +pathspec==0.12.1 +jupyter_core==5.1.0 +lit==18.1.8 +nvidia-curand-cu11==10.2.10.91 +nvidia-cublas-cu11==11.10.3.66 +nvidia-ml-py==12.560.30 +msgpack==1.1.0 +python-dateutil==2.8.2 +blessed==1.20.0 +packaging==23.0 +gitdb==4.0.11 +yarl==1.13.0 +emoji==2.8.0 +tzdata==2023.3 +cycler==0.12.1 +tornado==6.2 +backcall==0.2.0 +plotnine==0.12.4 +ninja==1.11.1.1 +latex==0.7.0 +wandb==0.18.5 +setproctitle==1.3.3 +threadpoolctl==3.2.0 +requests==2.32.3 +pyparsing==3.1.1 +smmap==5.0.1 +pyzmq==23.0.0 +async-timeout==4.0.3 +annotated-types==0.7.0 +matplotlib-inline==0.1.6 +latexcodec==1.0.0 +ipython==8.0.0 +patsy==0.5.3 +contourpy==1.2.0 +multidict==6.1.0 +mizani==0.9.3 +urllib3==2.1.0 +tokenizers==0.20.0 +MarkupSafe==2.1.2 +pip==24.2 +pexpect==4.8.0 +tqdm==4.66.5 +jedi==0.18.2 +pydantic_core==2.23.4 +tempdir==0.7.1 +mpmath==1.2.1 +setuptools==72.1.0 +pytest==7.4.3 +pure-eval==0.2.2 +psutil==5.9.1 +comm==0.1.2 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +regex==2023.10.3 +platformdirs==2.5.2 +asttokens==2.2.1 +torch==2.0.0 +nvidia-nccl-cu11==2.14.3 +xxhash==3.5.0 diff --git a/wandb/run-20241030_233740-a8ghkt6q/files/wandb-metadata.json b/wandb/run-20241030_233740-a8ghkt6q/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..a8dacbf09a4476768a1b912acd6e6aaa07e0b313 --- /dev/null +++ b/wandb/run-20241030_233740-a8ghkt6q/files/wandb-metadata.json @@ -0,0 +1,97 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-10-31T03:37:40.852967Z", + "args": [ + "--perturbation", + "reverse_control", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "3", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", + "codePath": "train/train_deep_wandb.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_deep_wandb.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA RTX A6000", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888559353856", + "used": "1711065919488" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + } + ], + "cudaVersion": "11.8" +} \ No newline at end of file diff --git a/wandb/run-20241030_233740-a8ghkt6q/logs/debug-internal.log b/wandb/run-20241030_233740-a8ghkt6q/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..ca006aa65623f99700ed02f024a13809894b0abc --- /dev/null +++ b/wandb/run-20241030_233740-a8ghkt6q/logs/debug-internal.log @@ -0,0 +1,8 @@ +{"time":"2024-10-30T23:37:40.855273129-04:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-10-30T23:37:40.855285669-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_233740-a8ghkt6q/logs/debug-core.log"} +{"time":"2024-10-30T23:37:40.96763436-04:00","level":"INFO","msg":"created new stream","id":"a8ghkt6q"} +{"time":"2024-10-30T23:37:40.967695611-04:00","level":"INFO","msg":"stream: started","id":"a8ghkt6q"} +{"time":"2024-10-30T23:37:40.967745251-04:00","level":"INFO","msg":"sender: started","stream_id":"a8ghkt6q"} +{"time":"2024-10-30T23:37:40.967712521-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"a8ghkt6q"}} +{"time":"2024-10-30T23:37:40.967732341-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"a8ghkt6q"}} +{"time":"2024-10-30T23:37:41.220876038-04:00","level":"INFO","msg":"Starting system monitor"} diff --git a/wandb/run-20241030_233740-a8ghkt6q/logs/debug.log b/wandb/run-20241030_233740-a8ghkt6q/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..7c08a1002924bfab47a13fde800ac829811db4b9 --- /dev/null +++ b/wandb/run-20241030_233740-a8ghkt6q/logs/debug.log @@ -0,0 +1,26 @@ +2024-10-30 23:37:40,849 INFO MainThread:464537 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-10-30 23:37:40,849 INFO MainThread:464537 [wandb_setup.py:_flush():79] Configure stats pid to 464537 +2024-10-30 23:37:40,849 INFO MainThread:464537 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-10-30 23:37:40,849 INFO MainThread:464537 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-10-30 23:37:40,849 INFO MainThread:464537 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-10-30 23:37:40,849 INFO MainThread:464537 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-10-30 23:37:40,849 INFO MainThread:464537 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'} +2024-10-30 23:37:40,849 INFO MainThread:464537 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-10-30 23:37:40,850 INFO MainThread:464537 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_233740-a8ghkt6q/logs/debug.log +2024-10-30 23:37:40,850 INFO MainThread:464537 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241030_233740-a8ghkt6q/logs/debug-internal.log +2024-10-30 23:37:40,850 INFO MainThread:464537 [wandb_init.py:init():621] calling init triggers +2024-10-30 23:37:40,850 INFO MainThread:464537 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-10-30 23:37:40,850 INFO MainThread:464537 [wandb_init.py:init():671] starting backend +2024-10-30 23:37:40,850 INFO MainThread:464537 [wandb_init.py:init():675] sending inform_init request +2024-10-30 23:37:40,852 INFO MainThread:464537 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-10-30 23:37:40,852 INFO MainThread:464537 [wandb_init.py:init():688] backend started and connected +2024-10-30 23:37:40,855 INFO MainThread:464537 [wandb_init.py:init():783] updated telemetry +2024-10-30 23:37:40,883 INFO MainThread:464537 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-10-30 23:37:41,217 INFO MainThread:464537 [wandb_init.py:init():867] starting run threads in backend +2024-10-30 23:37:42,837 INFO MainThread:464537 [wandb_run.py:_console_start():2463] atexit reg +2024-10-30 23:37:42,837 INFO MainThread:464537 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-10-30 23:37:42,837 INFO MainThread:464537 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-10-30 23:37:42,837 INFO MainThread:464537 [wandb_run.py:_redirect():2401] Redirects installed. +2024-10-30 23:37:42,860 INFO MainThread:464537 [wandb_init.py:init():911] run started, returning control to user process +2024-10-30 23:37:42,861 INFO MainThread:464537 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_control', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0} diff --git a/wandb/run-20241031_001055-dua2g15g/run-dua2g15g.wandb b/wandb/run-20241031_001055-dua2g15g/run-dua2g15g.wandb new file mode 100644 index 0000000000000000000000000000000000000000..d646c8f24536f3e06a5097af8ed5d7aab72e7306 --- /dev/null +++ b/wandb/run-20241031_001055-dua2g15g/run-dua2g15g.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1cf00c20fe4ac81c7ab3f9adde203aab4ee8211db895fe69b24a47b6d5b260f +size 327680 diff --git a/wandb/run-20241031_002020-qq5oimta/files/wandb-summary.json b/wandb/run-20241031_002020-qq5oimta/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..4f8a2d8f302b1f64c484c31f4742ad6275018b3d --- /dev/null +++ b/wandb/run-20241031_002020-qq5oimta/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":32219}} \ No newline at end of file diff --git a/wandb/run-20241031_002020-qq5oimta/logs/debug-internal.log b/wandb/run-20241031_002020-qq5oimta/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..2f8a0d31402b94d8e3339230ef1ce2cb1ef494be --- /dev/null +++ b/wandb/run-20241031_002020-qq5oimta/logs/debug-internal.log @@ -0,0 +1,20 @@ +{"time":"2024-10-31T00:20:20.034540173-04:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-10-31T00:20:20.034551713-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_002020-qq5oimta/logs/debug-core.log"} +{"time":"2024-10-31T00:20:20.142123016-04:00","level":"INFO","msg":"created new stream","id":"qq5oimta"} +{"time":"2024-10-31T00:20:20.142164126-04:00","level":"INFO","msg":"stream: started","id":"qq5oimta"} +{"time":"2024-10-31T00:20:20.142182576-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"qq5oimta"}} +{"time":"2024-10-31T00:20:20.142443509-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"qq5oimta"}} +{"time":"2024-10-31T00:20:20.142465839-04:00","level":"INFO","msg":"sender: started","stream_id":"qq5oimta"} +{"time":"2024-10-31T00:20:21.404732185-04:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-10-31T00:36:56.749060403-04:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"} +{"time":"2024-10-31T01:06:48.979811136-04:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/yaning1001-dartmouth-college/impossible_llm_reverse/qq5oimta/file_stream"} +{"time":"2024-10-31T06:09:10.032616886-04:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/yaning1001-dartmouth-college/impossible_llm_reverse/qq5oimta/file_stream"} +{"time":"2024-10-31T09:17:19.827022088-04:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2024-10-31T09:17:19.904160799-04:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2024-10-31T09:17:20.764779005-04:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"saving job artifact","runtime_seconds":0.617493792,"subtasks":[{"desc":"wandb-job.json","runtime_seconds":0.043146083,"progress":"563B/563B"}]}],"total_operations":1}} +{"time":"2024-10-31T09:17:21.720810984-04:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2024-10-31T09:17:22.833725492-04:00","level":"INFO","msg":"stream: closing","id":"qq5oimta"} +{"time":"2024-10-31T09:17:22.833769292-04:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"qq5oimta"}} +{"time":"2024-10-31T09:17:22.833834722-04:00","level":"INFO","msg":"sender: closed","stream_id":"qq5oimta"} +{"time":"2024-10-31T09:17:22.833829992-04:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"qq5oimta"}} +{"time":"2024-10-31T09:17:22.833996253-04:00","level":"INFO","msg":"stream: closed","id":"qq5oimta"} diff --git a/wandb/run-20241031_002020-qq5oimta/logs/debug.log b/wandb/run-20241031_002020-qq5oimta/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..19d4528d2a1449ef405b1009a1459e52454cc107 --- /dev/null +++ b/wandb/run-20241031_002020-qq5oimta/logs/debug.log @@ -0,0 +1,33 @@ +2024-10-31 00:20:20,030 INFO MainThread:484459 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-10-31 00:20:20,031 INFO MainThread:484459 [wandb_setup.py:_flush():79] Configure stats pid to 484459 +2024-10-31 00:20:20,031 INFO MainThread:484459 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-10-31 00:20:20,031 INFO MainThread:484459 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-10-31 00:20:20,031 INFO MainThread:484459 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-10-31 00:20:20,031 INFO MainThread:484459 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-10-31 00:20:20,031 INFO MainThread:484459 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'} +2024-10-31 00:20:20,031 INFO MainThread:484459 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-10-31 00:20:20,031 INFO MainThread:484459 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_002020-qq5oimta/logs/debug.log +2024-10-31 00:20:20,031 INFO MainThread:484459 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241031_002020-qq5oimta/logs/debug-internal.log +2024-10-31 00:20:20,031 INFO MainThread:484459 [wandb_init.py:init():621] calling init triggers +2024-10-31 00:20:20,031 INFO MainThread:484459 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-10-31 00:20:20,031 INFO MainThread:484459 [wandb_init.py:init():671] starting backend +2024-10-31 00:20:20,031 INFO MainThread:484459 [wandb_init.py:init():675] sending inform_init request +2024-10-31 00:20:20,032 INFO MainThread:484459 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-10-31 00:20:20,032 INFO MainThread:484459 [wandb_init.py:init():688] backend started and connected +2024-10-31 00:20:20,036 INFO MainThread:484459 [wandb_init.py:init():783] updated telemetry +2024-10-31 00:20:20,065 INFO MainThread:484459 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-10-31 00:20:21,401 INFO MainThread:484459 [wandb_init.py:init():867] starting run threads in backend +2024-10-31 00:20:21,535 INFO MainThread:484459 [wandb_run.py:_console_start():2463] atexit reg +2024-10-31 00:20:21,535 INFO MainThread:484459 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-10-31 00:20:21,535 INFO MainThread:484459 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-10-31 00:20:21,535 INFO MainThread:484459 [wandb_run.py:_redirect():2401] Redirects installed. +2024-10-31 00:20:21,537 INFO MainThread:484459 [wandb_init.py:init():911] run started, returning control to user process +2024-10-31 00:20:21,537 INFO MainThread:484459 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_full', 'train_set': '10M', 'batch_size': 3, 'epoch': 6, 'seed': 0, 'lr': 1e-05} +2024-10-31 09:17:19,684 INFO MainThread:484459 [wandb_run.py:_finish():2158] finishing run yaning1001-dartmouth-college/impossible_llm_reverse/qq5oimta +2024-10-31 09:17:19,700 INFO MainThread:484459 [wandb_run.py:_atexit_cleanup():2426] got exitcode: 0 +2024-10-31 09:17:19,748 INFO MainThread:484459 [wandb_run.py:_restore():2408] restore +2024-10-31 09:17:19,749 INFO MainThread:484459 [wandb_run.py:_restore():2414] restore done +2024-10-31 09:17:22,826 INFO MainThread:484459 [wandb_run.py:_footer_history_summary_info():3975] rendering history +2024-10-31 09:17:22,827 INFO MainThread:484459 [wandb_run.py:_footer_history_summary_info():4007] rendering summary +2024-10-31 09:17:22,833 INFO MainThread:484459 [wandb_run.py:_footer_sync_info():3934] logging synced files diff --git a/wandb/run-20241101_012438-qowf210g/logs/debug-internal.log b/wandb/run-20241101_012438-qowf210g/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..460e419cb7c855b528737908a29dc509577fd0a0 --- /dev/null +++ b/wandb/run-20241101_012438-qowf210g/logs/debug-internal.log @@ -0,0 +1,16 @@ +{"time":"2024-11-01T01:24:38.220929393-04:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-11-01T01:24:38.220939703-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_012438-qowf210g/logs/debug-core.log"} +{"time":"2024-11-01T01:24:38.327424494-04:00","level":"INFO","msg":"created new stream","id":"qowf210g"} +{"time":"2024-11-01T01:24:38.327466724-04:00","level":"INFO","msg":"stream: started","id":"qowf210g"} +{"time":"2024-11-01T01:24:38.327501474-04:00","level":"INFO","msg":"sender: started","stream_id":"qowf210g"} +{"time":"2024-11-01T01:24:38.327480694-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"qowf210g"}} +{"time":"2024-11-01T01:24:38.327492874-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"qowf210g"}} +{"time":"2024-11-01T01:24:38.529752791-04:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-11-01T01:24:38.641652449-04:00","level":"INFO","msg":"stream: closing","id":"qowf210g"} +{"time":"2024-11-01T01:24:38.64170344-04:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2024-11-01T01:24:38.661560348-04:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2024-11-01T01:24:39.235879964-04:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2024-11-01T01:24:39.374996001-04:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"qowf210g"}} +{"time":"2024-11-01T01:24:39.375064892-04:00","level":"INFO","msg":"sender: closed","stream_id":"qowf210g"} +{"time":"2024-11-01T01:24:39.375054422-04:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"qowf210g"}} +{"time":"2024-11-01T01:24:39.375140422-04:00","level":"INFO","msg":"stream: closed","id":"qowf210g"} diff --git a/wandb/run-20241101_012733-val6n9r9/files/output.log b/wandb/run-20241101_012733-val6n9r9/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..5b4e23864eac8a02b0874624fb2f33f4dc003e33 --- /dev/null +++ b/wandb/run-20241101_012733-val6n9r9/files/output.log @@ -0,0 +1,16 @@ +Downloading shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [02:32<00:00, 76.35s/it] +Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:06<00:00, 3.42s/it] +Map: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 16425/16425 [00:52<00:00, 310.38 examples/s] +Map: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 17013/17013 [00:49<00:00, 341.92 examples/s] +tokenized_valid: Dataset({ + features: ['input_ids', 'attention_mask'], + num_rows: 600 +}) +/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +[2024-11-01 01:32:26,372] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-11-01 01:32:36,756] [INFO] [comm.py:652:init_distributed] cdb=None +Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination +Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root... +Loading extension module cpu_adam... +Time to load cpu_adam op: 5.428602933883667 seconds diff --git a/wandb/run-20241101_012733-val6n9r9/files/requirements.txt b/wandb/run-20241101_012733-val6n9r9/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..95a931302e269cc9e4fa5b719b6511f176ee2416 --- /dev/null +++ b/wandb/run-20241101_012733-val6n9r9/files/requirements.txt @@ -0,0 +1,147 @@ +funcsigs==1.0.2 +sentry-sdk==2.17.0 +multiprocess==0.70.16 +numpy==1.26.2 +pluralizer==1.2.0 +debugpy==1.6.7 +nvidia-cudnn-cu11==8.5.0.96 +deepspeed==0.15.2 +data==0.4 +pandas==2.1.3 +tomli==2.0.1 +charset-normalizer==3.3.2 +attrs==24.2.0 +aiosignal==1.3.1 +fsspec==2023.10.0 +nvidia-cusparse-cu11==11.7.4.91 +zipp==3.12.0 +mypy-extensions==1.0.0 +datasets==3.0.1 +joblib==1.3.2 +hjson==3.1.0 +traitlets==5.7.1 +stack-data==0.6.0 +transformers==4.45.1 +sympy==1.11.1 +Pygments==2.15.0 +docker-pycreds==0.4.0 +dill==0.3.8 +wheel==0.44.0 +prompt-toolkit==3.0.30 +parso==0.8.3 +ipykernel==6.23.1 +pyarrow==17.0.0 +certifi==2023.11.17 +nvidia-cufft-cu11==10.9.0.58 +six==1.16.0 +pydantic==2.9.2 +click==8.1.7 +nest-asyncio==1.5.6 +gmpy2==2.1.0 +matplotlib==3.8.2 +scipy==1.11.4 +typing_extensions==4.12.2 +statsmodels==0.14.0 +huggingface-hub==0.25.0 +frozenlist==1.4.1 +gpustat==1.1.1 +nvidia-nvtx-cu11==11.7.91 +safetensors==0.4.5 +stanza==1.9.2 +decorator==5.1.1 +seaborn==0.13.0 +sentencepiece==0.2.0 +PyYAML==6.0.1 +black==24.8.0 +protobuf==4.25.1 +pickleshare==0.7.5 +peft==0.13.0 +triton==2.0.0 +nvidia-cuda-runtime-cu11==11.7.99 +Jinja2==3.1.2 +nvidia-cusolver-cu11==11.4.0.1 +executing==1.2.0 +jupyter_client==8.1.0 +pluggy==1.3.0 +cmake==3.30.3 +pytz==2023.3.post1 +aiohappyeyeballs==2.4.2 +kiwisolver==1.4.5 +py-cpuinfo==9.0.0 +Pillow==10.1.0 +ptyprocess==0.7.0 +importlib_resources==6.4.5 +GitPython==3.1.43 +importlib-metadata==6.0.0 +iniconfig==2.0.0 +scikit-learn==1.3.2 +exceptiongroup==1.1.0 +networkx==2.8.6 +accelerate==1.0.0 +nltk==3.8.1 +shutilwhich==1.1.0 +fonttools==4.45.1 +future==0.18.3 +aiohttp==3.10.6 +wcwidth==0.2.5 +idna==3.6 +filelock==3.12.2 +pathspec==0.12.1 +jupyter_core==5.1.0 +lit==18.1.8 +nvidia-curand-cu11==10.2.10.91 +nvidia-cublas-cu11==11.10.3.66 +nvidia-ml-py==12.560.30 +msgpack==1.1.0 +python-dateutil==2.8.2 +blessed==1.20.0 +packaging==23.0 +gitdb==4.0.11 +yarl==1.13.0 +emoji==2.8.0 +tzdata==2023.3 +cycler==0.12.1 +tornado==6.2 +backcall==0.2.0 +plotnine==0.12.4 +ninja==1.11.1.1 +latex==0.7.0 +wandb==0.18.5 +setproctitle==1.3.3 +threadpoolctl==3.2.0 +requests==2.32.3 +pyparsing==3.1.1 +smmap==5.0.1 +pyzmq==23.0.0 +async-timeout==4.0.3 +annotated-types==0.7.0 +matplotlib-inline==0.1.6 +latexcodec==1.0.0 +ipython==8.0.0 +patsy==0.5.3 +contourpy==1.2.0 +multidict==6.1.0 +mizani==0.9.3 +urllib3==2.1.0 +tokenizers==0.20.0 +MarkupSafe==2.1.2 +pip==24.2 +pexpect==4.8.0 +tqdm==4.66.5 +jedi==0.18.2 +pydantic_core==2.23.4 +tempdir==0.7.1 +mpmath==1.2.1 +setuptools==72.1.0 +pytest==7.4.3 +pure-eval==0.2.2 +psutil==5.9.1 +comm==0.1.2 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +regex==2023.10.3 +platformdirs==2.5.2 +asttokens==2.2.1 +torch==2.0.0 +nvidia-nccl-cu11==2.14.3 +xxhash==3.5.0 diff --git a/wandb/run-20241101_012733-val6n9r9/files/wandb-metadata.json b/wandb/run-20241101_012733-val6n9r9/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..9b2223813cdb5c6a3ac9a121bac1324844446e19 --- /dev/null +++ b/wandb/run-20241101_012733-val6n9r9/files/wandb-metadata.json @@ -0,0 +1,97 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-11-01T05:27:33.992750Z", + "args": [ + "--perturbation", + "shuffle_nondeterministic", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "6", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", + "codePath": "train/train_deep_wandb.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_deep_wandb.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA RTX A6000", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888559353856", + "used": "1753992269824" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + } + ], + "cudaVersion": "11.8" +} \ No newline at end of file diff --git a/wandb/run-20241101_012733-val6n9r9/logs/debug-internal.log b/wandb/run-20241101_012733-val6n9r9/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..7d87d757400f2274061db0bc59e06b40e03576ba --- /dev/null +++ b/wandb/run-20241101_012733-val6n9r9/logs/debug-internal.log @@ -0,0 +1,8 @@ +{"time":"2024-11-01T01:27:33.995250587-04:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-11-01T01:27:33.995267057-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_012733-val6n9r9/logs/debug-core.log"} +{"time":"2024-11-01T01:27:34.1039281-04:00","level":"INFO","msg":"created new stream","id":"val6n9r9"} +{"time":"2024-11-01T01:27:34.103978121-04:00","level":"INFO","msg":"stream: started","id":"val6n9r9"} +{"time":"2024-11-01T01:27:34.104094092-04:00","level":"INFO","msg":"sender: started","stream_id":"val6n9r9"} +{"time":"2024-11-01T01:27:34.104027291-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"val6n9r9"}} +{"time":"2024-11-01T01:27:34.104049701-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"val6n9r9"}} +{"time":"2024-11-01T01:27:34.310274622-04:00","level":"INFO","msg":"Starting system monitor"} diff --git a/wandb/run-20241101_012733-val6n9r9/logs/debug.log b/wandb/run-20241101_012733-val6n9r9/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..860d7a9a005dc6df59c14fa6bda2606ee1cad64a --- /dev/null +++ b/wandb/run-20241101_012733-val6n9r9/logs/debug.log @@ -0,0 +1,26 @@ +2024-11-01 01:27:33,990 INFO MainThread:678553 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-11-01 01:27:33,990 INFO MainThread:678553 [wandb_setup.py:_flush():79] Configure stats pid to 678553 +2024-11-01 01:27:33,990 INFO MainThread:678553 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-11-01 01:27:33,990 INFO MainThread:678553 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-11-01 01:27:33,990 INFO MainThread:678553 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-01 01:27:33,990 INFO MainThread:678553 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-01 01:27:33,990 INFO MainThread:678553 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'} +2024-11-01 01:27:33,990 INFO MainThread:678553 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-01 01:27:33,990 INFO MainThread:678553 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_012733-val6n9r9/logs/debug.log +2024-11-01 01:27:33,990 INFO MainThread:678553 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_012733-val6n9r9/logs/debug-internal.log +2024-11-01 01:27:33,991 INFO MainThread:678553 [wandb_init.py:init():621] calling init triggers +2024-11-01 01:27:33,991 INFO MainThread:678553 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-11-01 01:27:33,991 INFO MainThread:678553 [wandb_init.py:init():671] starting backend +2024-11-01 01:27:33,991 INFO MainThread:678553 [wandb_init.py:init():675] sending inform_init request +2024-11-01 01:27:33,992 INFO MainThread:678553 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-01 01:27:33,992 INFO MainThread:678553 [wandb_init.py:init():688] backend started and connected +2024-11-01 01:27:33,996 INFO MainThread:678553 [wandb_init.py:init():783] updated telemetry +2024-11-01 01:27:34,020 INFO MainThread:678553 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-11-01 01:27:34,307 INFO MainThread:678553 [wandb_init.py:init():867] starting run threads in backend +2024-11-01 01:27:34,397 INFO MainThread:678553 [wandb_run.py:_console_start():2463] atexit reg +2024-11-01 01:27:34,397 INFO MainThread:678553 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-11-01 01:27:34,397 INFO MainThread:678553 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-11-01 01:27:34,397 INFO MainThread:678553 [wandb_run.py:_redirect():2401] Redirects installed. +2024-11-01 01:27:34,399 INFO MainThread:678553 [wandb_init.py:init():911] run started, returning control to user process +2024-11-01 01:27:34,399 INFO MainThread:678553 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_nondeterministic', 'train_set': '10M', 'batch_size': 3, 'epoch': 6, 'seed': 0, 'lr': 5e-06} diff --git a/wandb/run-20241101_093116-jo652wfc/files/output.log b/wandb/run-20241101_093116-jo652wfc/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..6992c3ac596adf58b5d6948e50c81e3bfb8f9c37 --- /dev/null +++ b/wandb/run-20241101_093116-jo652wfc/files/output.log @@ -0,0 +1,13 @@ +Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:18<00:00, 9.08s/it] +tokenized_valid: Dataset({ + features: ['input_ids', 'attention_mask'], + num_rows: 600 +}) +/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +[2024-11-01 09:31:37,056] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-11-01 09:31:45,780] [INFO] [comm.py:652:init_distributed] cdb=None +Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination +Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root... +Loading extension module cpu_adam... +Time to load cpu_adam op: 4.828465461730957 seconds diff --git a/wandb/run-20241101_093116-jo652wfc/files/requirements.txt b/wandb/run-20241101_093116-jo652wfc/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..95a931302e269cc9e4fa5b719b6511f176ee2416 --- /dev/null +++ b/wandb/run-20241101_093116-jo652wfc/files/requirements.txt @@ -0,0 +1,147 @@ +funcsigs==1.0.2 +sentry-sdk==2.17.0 +multiprocess==0.70.16 +numpy==1.26.2 +pluralizer==1.2.0 +debugpy==1.6.7 +nvidia-cudnn-cu11==8.5.0.96 +deepspeed==0.15.2 +data==0.4 +pandas==2.1.3 +tomli==2.0.1 +charset-normalizer==3.3.2 +attrs==24.2.0 +aiosignal==1.3.1 +fsspec==2023.10.0 +nvidia-cusparse-cu11==11.7.4.91 +zipp==3.12.0 +mypy-extensions==1.0.0 +datasets==3.0.1 +joblib==1.3.2 +hjson==3.1.0 +traitlets==5.7.1 +stack-data==0.6.0 +transformers==4.45.1 +sympy==1.11.1 +Pygments==2.15.0 +docker-pycreds==0.4.0 +dill==0.3.8 +wheel==0.44.0 +prompt-toolkit==3.0.30 +parso==0.8.3 +ipykernel==6.23.1 +pyarrow==17.0.0 +certifi==2023.11.17 +nvidia-cufft-cu11==10.9.0.58 +six==1.16.0 +pydantic==2.9.2 +click==8.1.7 +nest-asyncio==1.5.6 +gmpy2==2.1.0 +matplotlib==3.8.2 +scipy==1.11.4 +typing_extensions==4.12.2 +statsmodels==0.14.0 +huggingface-hub==0.25.0 +frozenlist==1.4.1 +gpustat==1.1.1 +nvidia-nvtx-cu11==11.7.91 +safetensors==0.4.5 +stanza==1.9.2 +decorator==5.1.1 +seaborn==0.13.0 +sentencepiece==0.2.0 +PyYAML==6.0.1 +black==24.8.0 +protobuf==4.25.1 +pickleshare==0.7.5 +peft==0.13.0 +triton==2.0.0 +nvidia-cuda-runtime-cu11==11.7.99 +Jinja2==3.1.2 +nvidia-cusolver-cu11==11.4.0.1 +executing==1.2.0 +jupyter_client==8.1.0 +pluggy==1.3.0 +cmake==3.30.3 +pytz==2023.3.post1 +aiohappyeyeballs==2.4.2 +kiwisolver==1.4.5 +py-cpuinfo==9.0.0 +Pillow==10.1.0 +ptyprocess==0.7.0 +importlib_resources==6.4.5 +GitPython==3.1.43 +importlib-metadata==6.0.0 +iniconfig==2.0.0 +scikit-learn==1.3.2 +exceptiongroup==1.1.0 +networkx==2.8.6 +accelerate==1.0.0 +nltk==3.8.1 +shutilwhich==1.1.0 +fonttools==4.45.1 +future==0.18.3 +aiohttp==3.10.6 +wcwidth==0.2.5 +idna==3.6 +filelock==3.12.2 +pathspec==0.12.1 +jupyter_core==5.1.0 +lit==18.1.8 +nvidia-curand-cu11==10.2.10.91 +nvidia-cublas-cu11==11.10.3.66 +nvidia-ml-py==12.560.30 +msgpack==1.1.0 +python-dateutil==2.8.2 +blessed==1.20.0 +packaging==23.0 +gitdb==4.0.11 +yarl==1.13.0 +emoji==2.8.0 +tzdata==2023.3 +cycler==0.12.1 +tornado==6.2 +backcall==0.2.0 +plotnine==0.12.4 +ninja==1.11.1.1 +latex==0.7.0 +wandb==0.18.5 +setproctitle==1.3.3 +threadpoolctl==3.2.0 +requests==2.32.3 +pyparsing==3.1.1 +smmap==5.0.1 +pyzmq==23.0.0 +async-timeout==4.0.3 +annotated-types==0.7.0 +matplotlib-inline==0.1.6 +latexcodec==1.0.0 +ipython==8.0.0 +patsy==0.5.3 +contourpy==1.2.0 +multidict==6.1.0 +mizani==0.9.3 +urllib3==2.1.0 +tokenizers==0.20.0 +MarkupSafe==2.1.2 +pip==24.2 +pexpect==4.8.0 +tqdm==4.66.5 +jedi==0.18.2 +pydantic_core==2.23.4 +tempdir==0.7.1 +mpmath==1.2.1 +setuptools==72.1.0 +pytest==7.4.3 +pure-eval==0.2.2 +psutil==5.9.1 +comm==0.1.2 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +regex==2023.10.3 +platformdirs==2.5.2 +asttokens==2.2.1 +torch==2.0.0 +nvidia-nccl-cu11==2.14.3 +xxhash==3.5.0 diff --git a/wandb/run-20241101_093116-jo652wfc/files/wandb-metadata.json b/wandb/run-20241101_093116-jo652wfc/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..1b79b3956131a21c1df0746e8e4675fbe9155484 --- /dev/null +++ b/wandb/run-20241101_093116-jo652wfc/files/wandb-metadata.json @@ -0,0 +1,97 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-11-01T13:31:16.509088Z", + "args": [ + "--perturbation", + "reverse_control", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "7", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", + "codePath": "train/train_deep_wandb.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_deep_wandb.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA RTX A6000", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888559353856", + "used": "1754716262400" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + } + ], + "cudaVersion": "11.8" +} \ No newline at end of file diff --git a/wandb/run-20241101_093116-jo652wfc/logs/debug-internal.log b/wandb/run-20241101_093116-jo652wfc/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..a65d6c768e5069a91fdfea606e1c37a33059d66b --- /dev/null +++ b/wandb/run-20241101_093116-jo652wfc/logs/debug-internal.log @@ -0,0 +1,8 @@ +{"time":"2024-11-01T09:31:16.511149419-04:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-11-01T09:31:16.511162399-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_093116-jo652wfc/logs/debug-core.log"} +{"time":"2024-11-01T09:31:16.618441589-04:00","level":"INFO","msg":"created new stream","id":"jo652wfc"} +{"time":"2024-11-01T09:31:16.618474009-04:00","level":"INFO","msg":"stream: started","id":"jo652wfc"} +{"time":"2024-11-01T09:31:16.618503669-04:00","level":"INFO","msg":"sender: started","stream_id":"jo652wfc"} +{"time":"2024-11-01T09:31:16.618487059-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"jo652wfc"}} +{"time":"2024-11-01T09:31:16.618502009-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"jo652wfc"}} +{"time":"2024-11-01T09:31:16.832438843-04:00","level":"INFO","msg":"Starting system monitor"} diff --git a/wandb/run-20241101_093116-jo652wfc/logs/debug.log b/wandb/run-20241101_093116-jo652wfc/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b9e052596d8fc796249240700a08a1f7c52ac13b --- /dev/null +++ b/wandb/run-20241101_093116-jo652wfc/logs/debug.log @@ -0,0 +1,26 @@ +2024-11-01 09:31:16,507 INFO MainThread:781949 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-11-01 09:31:16,507 INFO MainThread:781949 [wandb_setup.py:_flush():79] Configure stats pid to 781949 +2024-11-01 09:31:16,507 INFO MainThread:781949 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-11-01 09:31:16,507 INFO MainThread:781949 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-11-01 09:31:16,507 INFO MainThread:781949 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-01 09:31:16,507 INFO MainThread:781949 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-01 09:31:16,507 INFO MainThread:781949 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'} +2024-11-01 09:31:16,507 INFO MainThread:781949 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-01 09:31:16,507 INFO MainThread:781949 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_093116-jo652wfc/logs/debug.log +2024-11-01 09:31:16,507 INFO MainThread:781949 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_093116-jo652wfc/logs/debug-internal.log +2024-11-01 09:31:16,507 INFO MainThread:781949 [wandb_init.py:init():621] calling init triggers +2024-11-01 09:31:16,507 INFO MainThread:781949 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-11-01 09:31:16,507 INFO MainThread:781949 [wandb_init.py:init():671] starting backend +2024-11-01 09:31:16,507 INFO MainThread:781949 [wandb_init.py:init():675] sending inform_init request +2024-11-01 09:31:16,508 INFO MainThread:781949 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-01 09:31:16,508 INFO MainThread:781949 [wandb_init.py:init():688] backend started and connected +2024-11-01 09:31:16,511 INFO MainThread:781949 [wandb_init.py:init():783] updated telemetry +2024-11-01 09:31:16,539 INFO MainThread:781949 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-11-01 09:31:16,829 INFO MainThread:781949 [wandb_init.py:init():867] starting run threads in backend +2024-11-01 09:31:16,925 INFO MainThread:781949 [wandb_run.py:_console_start():2463] atexit reg +2024-11-01 09:31:16,925 INFO MainThread:781949 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-11-01 09:31:16,925 INFO MainThread:781949 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-11-01 09:31:16,925 INFO MainThread:781949 [wandb_run.py:_redirect():2401] Redirects installed. +2024-11-01 09:31:16,927 INFO MainThread:781949 [wandb_init.py:init():911] run started, returning control to user process +2024-11-01 09:31:16,927 INFO MainThread:781949 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_control', 'train_set': '10M', 'batch_size': 3, 'epoch': 7, 'seed': 0, 'lr': 5e-06} diff --git a/wandb/run-20241101_093116-jo652wfc/run-jo652wfc.wandb b/wandb/run-20241101_093116-jo652wfc/run-jo652wfc.wandb new file mode 100644 index 0000000000000000000000000000000000000000..da9be571313f8f48bd3d9a6911999ed04dfc1ea7 Binary files /dev/null and b/wandb/run-20241101_093116-jo652wfc/run-jo652wfc.wandb differ diff --git a/wandb/run-20241101_093116-w11cgu13/files/output.log b/wandb/run-20241101_093116-w11cgu13/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..1e640ae0ea5f32873b0f90eb741b9c8e8c5fa395 --- /dev/null +++ b/wandb/run-20241101_093116-w11cgu13/files/output.log @@ -0,0 +1,13 @@ +Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:18<00:00, 9.04s/it] +tokenized_valid: Dataset({ + features: ['input_ids', 'attention_mask'], + num_rows: 600 +}) +/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +[2024-11-01 09:31:37,040] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-11-01 09:31:45,693] [INFO] [comm.py:652:init_distributed] cdb=None +Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination +Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root... +Loading extension module cpu_adam... +Time to load cpu_adam op: 4.85305643081665 seconds diff --git a/wandb/run-20241101_093116-w11cgu13/files/requirements.txt b/wandb/run-20241101_093116-w11cgu13/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..95a931302e269cc9e4fa5b719b6511f176ee2416 --- /dev/null +++ b/wandb/run-20241101_093116-w11cgu13/files/requirements.txt @@ -0,0 +1,147 @@ +funcsigs==1.0.2 +sentry-sdk==2.17.0 +multiprocess==0.70.16 +numpy==1.26.2 +pluralizer==1.2.0 +debugpy==1.6.7 +nvidia-cudnn-cu11==8.5.0.96 +deepspeed==0.15.2 +data==0.4 +pandas==2.1.3 +tomli==2.0.1 +charset-normalizer==3.3.2 +attrs==24.2.0 +aiosignal==1.3.1 +fsspec==2023.10.0 +nvidia-cusparse-cu11==11.7.4.91 +zipp==3.12.0 +mypy-extensions==1.0.0 +datasets==3.0.1 +joblib==1.3.2 +hjson==3.1.0 +traitlets==5.7.1 +stack-data==0.6.0 +transformers==4.45.1 +sympy==1.11.1 +Pygments==2.15.0 +docker-pycreds==0.4.0 +dill==0.3.8 +wheel==0.44.0 +prompt-toolkit==3.0.30 +parso==0.8.3 +ipykernel==6.23.1 +pyarrow==17.0.0 +certifi==2023.11.17 +nvidia-cufft-cu11==10.9.0.58 +six==1.16.0 +pydantic==2.9.2 +click==8.1.7 +nest-asyncio==1.5.6 +gmpy2==2.1.0 +matplotlib==3.8.2 +scipy==1.11.4 +typing_extensions==4.12.2 +statsmodels==0.14.0 +huggingface-hub==0.25.0 +frozenlist==1.4.1 +gpustat==1.1.1 +nvidia-nvtx-cu11==11.7.91 +safetensors==0.4.5 +stanza==1.9.2 +decorator==5.1.1 +seaborn==0.13.0 +sentencepiece==0.2.0 +PyYAML==6.0.1 +black==24.8.0 +protobuf==4.25.1 +pickleshare==0.7.5 +peft==0.13.0 +triton==2.0.0 +nvidia-cuda-runtime-cu11==11.7.99 +Jinja2==3.1.2 +nvidia-cusolver-cu11==11.4.0.1 +executing==1.2.0 +jupyter_client==8.1.0 +pluggy==1.3.0 +cmake==3.30.3 +pytz==2023.3.post1 +aiohappyeyeballs==2.4.2 +kiwisolver==1.4.5 +py-cpuinfo==9.0.0 +Pillow==10.1.0 +ptyprocess==0.7.0 +importlib_resources==6.4.5 +GitPython==3.1.43 +importlib-metadata==6.0.0 +iniconfig==2.0.0 +scikit-learn==1.3.2 +exceptiongroup==1.1.0 +networkx==2.8.6 +accelerate==1.0.0 +nltk==3.8.1 +shutilwhich==1.1.0 +fonttools==4.45.1 +future==0.18.3 +aiohttp==3.10.6 +wcwidth==0.2.5 +idna==3.6 +filelock==3.12.2 +pathspec==0.12.1 +jupyter_core==5.1.0 +lit==18.1.8 +nvidia-curand-cu11==10.2.10.91 +nvidia-cublas-cu11==11.10.3.66 +nvidia-ml-py==12.560.30 +msgpack==1.1.0 +python-dateutil==2.8.2 +blessed==1.20.0 +packaging==23.0 +gitdb==4.0.11 +yarl==1.13.0 +emoji==2.8.0 +tzdata==2023.3 +cycler==0.12.1 +tornado==6.2 +backcall==0.2.0 +plotnine==0.12.4 +ninja==1.11.1.1 +latex==0.7.0 +wandb==0.18.5 +setproctitle==1.3.3 +threadpoolctl==3.2.0 +requests==2.32.3 +pyparsing==3.1.1 +smmap==5.0.1 +pyzmq==23.0.0 +async-timeout==4.0.3 +annotated-types==0.7.0 +matplotlib-inline==0.1.6 +latexcodec==1.0.0 +ipython==8.0.0 +patsy==0.5.3 +contourpy==1.2.0 +multidict==6.1.0 +mizani==0.9.3 +urllib3==2.1.0 +tokenizers==0.20.0 +MarkupSafe==2.1.2 +pip==24.2 +pexpect==4.8.0 +tqdm==4.66.5 +jedi==0.18.2 +pydantic_core==2.23.4 +tempdir==0.7.1 +mpmath==1.2.1 +setuptools==72.1.0 +pytest==7.4.3 +pure-eval==0.2.2 +psutil==5.9.1 +comm==0.1.2 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +regex==2023.10.3 +platformdirs==2.5.2 +asttokens==2.2.1 +torch==2.0.0 +nvidia-nccl-cu11==2.14.3 +xxhash==3.5.0 diff --git a/wandb/run-20241101_093116-w11cgu13/files/wandb-metadata.json b/wandb/run-20241101_093116-w11cgu13/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..567e5fc3294af7a6d548699af0f1dce71a7fe9d3 --- /dev/null +++ b/wandb/run-20241101_093116-w11cgu13/files/wandb-metadata.json @@ -0,0 +1,97 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-11-01T13:31:16.513694Z", + "args": [ + "--perturbation", + "reverse_control", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "7", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", + "codePath": "train/train_deep_wandb.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_deep_wandb.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA RTX A6000", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888559353856", + "used": "1754716262400" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + } + ], + "cudaVersion": "11.8" +} \ No newline at end of file diff --git a/wandb/run-20241101_093116-w11cgu13/logs/debug-internal.log b/wandb/run-20241101_093116-w11cgu13/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..abfa0ba60f2739fa72e97472c8ecbec3124b3d37 --- /dev/null +++ b/wandb/run-20241101_093116-w11cgu13/logs/debug-internal.log @@ -0,0 +1,8 @@ +{"time":"2024-11-01T09:31:16.515903771-04:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-11-01T09:31:16.515917391-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_093116-w11cgu13/logs/debug-core.log"} +{"time":"2024-11-01T09:31:16.623731793-04:00","level":"INFO","msg":"created new stream","id":"w11cgu13"} +{"time":"2024-11-01T09:31:16.623771704-04:00","level":"INFO","msg":"stream: started","id":"w11cgu13"} +{"time":"2024-11-01T09:31:16.623799034-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"w11cgu13"}} +{"time":"2024-11-01T09:31:16.623856324-04:00","level":"INFO","msg":"sender: started","stream_id":"w11cgu13"} +{"time":"2024-11-01T09:31:16.623831334-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"w11cgu13"}} +{"time":"2024-11-01T09:31:16.852066303-04:00","level":"INFO","msg":"Starting system monitor"} diff --git a/wandb/run-20241101_093116-w11cgu13/logs/debug.log b/wandb/run-20241101_093116-w11cgu13/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..447c5c4abe6d59f38d1cd77ed821923733d571fb --- /dev/null +++ b/wandb/run-20241101_093116-w11cgu13/logs/debug.log @@ -0,0 +1,26 @@ +2024-11-01 09:31:16,511 INFO MainThread:781946 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-11-01 09:31:16,511 INFO MainThread:781946 [wandb_setup.py:_flush():79] Configure stats pid to 781946 +2024-11-01 09:31:16,511 INFO MainThread:781946 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-11-01 09:31:16,511 INFO MainThread:781946 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-11-01 09:31:16,511 INFO MainThread:781946 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-01 09:31:16,511 INFO MainThread:781946 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-01 09:31:16,511 INFO MainThread:781946 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'} +2024-11-01 09:31:16,511 INFO MainThread:781946 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-01 09:31:16,512 INFO MainThread:781946 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_093116-w11cgu13/logs/debug.log +2024-11-01 09:31:16,512 INFO MainThread:781946 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_093116-w11cgu13/logs/debug-internal.log +2024-11-01 09:31:16,512 INFO MainThread:781946 [wandb_init.py:init():621] calling init triggers +2024-11-01 09:31:16,512 INFO MainThread:781946 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-11-01 09:31:16,512 INFO MainThread:781946 [wandb_init.py:init():671] starting backend +2024-11-01 09:31:16,512 INFO MainThread:781946 [wandb_init.py:init():675] sending inform_init request +2024-11-01 09:31:16,513 INFO MainThread:781946 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-01 09:31:16,513 INFO MainThread:781946 [wandb_init.py:init():688] backend started and connected +2024-11-01 09:31:16,516 INFO MainThread:781946 [wandb_init.py:init():783] updated telemetry +2024-11-01 09:31:16,551 INFO MainThread:781946 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-11-01 09:31:16,849 INFO MainThread:781946 [wandb_init.py:init():867] starting run threads in backend +2024-11-01 09:31:16,943 INFO MainThread:781946 [wandb_run.py:_console_start():2463] atexit reg +2024-11-01 09:31:16,943 INFO MainThread:781946 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-11-01 09:31:16,943 INFO MainThread:781946 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-11-01 09:31:16,943 INFO MainThread:781946 [wandb_run.py:_redirect():2401] Redirects installed. +2024-11-01 09:31:16,945 INFO MainThread:781946 [wandb_init.py:init():911] run started, returning control to user process +2024-11-01 09:31:16,945 INFO MainThread:781946 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_control', 'train_set': '10M', 'batch_size': 3, 'epoch': 7, 'seed': 0, 'lr': 5e-06} diff --git a/wandb/run-20241101_093116-w11cgu13/run-w11cgu13.wandb b/wandb/run-20241101_093116-w11cgu13/run-w11cgu13.wandb new file mode 100644 index 0000000000000000000000000000000000000000..c07257680a68e132e9aedd7d7b1e447984d65cea Binary files /dev/null and b/wandb/run-20241101_093116-w11cgu13/run-w11cgu13.wandb differ diff --git a/wandb/run-20241101_200535-lnp8ii96/files/output.log b/wandb/run-20241101_200535-lnp8ii96/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..a39c2a98abeb87e8a33dfb6cc99d24816940e19f --- /dev/null +++ b/wandb/run-20241101_200535-lnp8ii96/files/output.log @@ -0,0 +1,17 @@ +model-00001-of-00002.safetensors: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4.97G/4.97G [01:51<00:00, 42.0MB/s] +model-00002-of-00002.safetensors: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1.46G/1.46G [00:34<00:00, 42.3MB/s] +Downloading shards: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [02:25<00:00, 72.94s/it] +Loading checkpoint shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:05<00:00, 2.56s/it] +generation_config.json: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 185/185 [00:00<00:00, 66.7kB/s] +tokenized_valid: Dataset({ + features: ['input_ids', 'attention_mask'], + num_rows: 600 +}) +/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +[2024-11-01 20:08:09,443] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-11-01 20:08:19,367] [INFO] [comm.py:652:init_distributed] cdb=None +Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination +Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root... +Loading extension module cpu_adam... +Time to load cpu_adam op: 5.52386474609375 seconds diff --git a/wandb/run-20241101_200535-lnp8ii96/files/requirements.txt b/wandb/run-20241101_200535-lnp8ii96/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..95a931302e269cc9e4fa5b719b6511f176ee2416 --- /dev/null +++ b/wandb/run-20241101_200535-lnp8ii96/files/requirements.txt @@ -0,0 +1,147 @@ +funcsigs==1.0.2 +sentry-sdk==2.17.0 +multiprocess==0.70.16 +numpy==1.26.2 +pluralizer==1.2.0 +debugpy==1.6.7 +nvidia-cudnn-cu11==8.5.0.96 +deepspeed==0.15.2 +data==0.4 +pandas==2.1.3 +tomli==2.0.1 +charset-normalizer==3.3.2 +attrs==24.2.0 +aiosignal==1.3.1 +fsspec==2023.10.0 +nvidia-cusparse-cu11==11.7.4.91 +zipp==3.12.0 +mypy-extensions==1.0.0 +datasets==3.0.1 +joblib==1.3.2 +hjson==3.1.0 +traitlets==5.7.1 +stack-data==0.6.0 +transformers==4.45.1 +sympy==1.11.1 +Pygments==2.15.0 +docker-pycreds==0.4.0 +dill==0.3.8 +wheel==0.44.0 +prompt-toolkit==3.0.30 +parso==0.8.3 +ipykernel==6.23.1 +pyarrow==17.0.0 +certifi==2023.11.17 +nvidia-cufft-cu11==10.9.0.58 +six==1.16.0 +pydantic==2.9.2 +click==8.1.7 +nest-asyncio==1.5.6 +gmpy2==2.1.0 +matplotlib==3.8.2 +scipy==1.11.4 +typing_extensions==4.12.2 +statsmodels==0.14.0 +huggingface-hub==0.25.0 +frozenlist==1.4.1 +gpustat==1.1.1 +nvidia-nvtx-cu11==11.7.91 +safetensors==0.4.5 +stanza==1.9.2 +decorator==5.1.1 +seaborn==0.13.0 +sentencepiece==0.2.0 +PyYAML==6.0.1 +black==24.8.0 +protobuf==4.25.1 +pickleshare==0.7.5 +peft==0.13.0 +triton==2.0.0 +nvidia-cuda-runtime-cu11==11.7.99 +Jinja2==3.1.2 +nvidia-cusolver-cu11==11.4.0.1 +executing==1.2.0 +jupyter_client==8.1.0 +pluggy==1.3.0 +cmake==3.30.3 +pytz==2023.3.post1 +aiohappyeyeballs==2.4.2 +kiwisolver==1.4.5 +py-cpuinfo==9.0.0 +Pillow==10.1.0 +ptyprocess==0.7.0 +importlib_resources==6.4.5 +GitPython==3.1.43 +importlib-metadata==6.0.0 +iniconfig==2.0.0 +scikit-learn==1.3.2 +exceptiongroup==1.1.0 +networkx==2.8.6 +accelerate==1.0.0 +nltk==3.8.1 +shutilwhich==1.1.0 +fonttools==4.45.1 +future==0.18.3 +aiohttp==3.10.6 +wcwidth==0.2.5 +idna==3.6 +filelock==3.12.2 +pathspec==0.12.1 +jupyter_core==5.1.0 +lit==18.1.8 +nvidia-curand-cu11==10.2.10.91 +nvidia-cublas-cu11==11.10.3.66 +nvidia-ml-py==12.560.30 +msgpack==1.1.0 +python-dateutil==2.8.2 +blessed==1.20.0 +packaging==23.0 +gitdb==4.0.11 +yarl==1.13.0 +emoji==2.8.0 +tzdata==2023.3 +cycler==0.12.1 +tornado==6.2 +backcall==0.2.0 +plotnine==0.12.4 +ninja==1.11.1.1 +latex==0.7.0 +wandb==0.18.5 +setproctitle==1.3.3 +threadpoolctl==3.2.0 +requests==2.32.3 +pyparsing==3.1.1 +smmap==5.0.1 +pyzmq==23.0.0 +async-timeout==4.0.3 +annotated-types==0.7.0 +matplotlib-inline==0.1.6 +latexcodec==1.0.0 +ipython==8.0.0 +patsy==0.5.3 +contourpy==1.2.0 +multidict==6.1.0 +mizani==0.9.3 +urllib3==2.1.0 +tokenizers==0.20.0 +MarkupSafe==2.1.2 +pip==24.2 +pexpect==4.8.0 +tqdm==4.66.5 +jedi==0.18.2 +pydantic_core==2.23.4 +tempdir==0.7.1 +mpmath==1.2.1 +setuptools==72.1.0 +pytest==7.4.3 +pure-eval==0.2.2 +psutil==5.9.1 +comm==0.1.2 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +regex==2023.10.3 +platformdirs==2.5.2 +asttokens==2.2.1 +torch==2.0.0 +nvidia-nccl-cu11==2.14.3 +xxhash==3.5.0 diff --git a/wandb/run-20241101_200535-lnp8ii96/files/wandb-metadata.json b/wandb/run-20241101_200535-lnp8ii96/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..7e91e01928f65b8bfc3658f6ecf53986bd578847 --- /dev/null +++ b/wandb/run-20241101_200535-lnp8ii96/files/wandb-metadata.json @@ -0,0 +1,97 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-11-02T00:05:35.623567Z", + "args": [ + "--perturbation", + "shuffle_nondeterministic", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "3", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", + "codePath": "train/train_deep_wandb.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_deep_wandb.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA RTX A6000", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888559353856", + "used": "1754801684480" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + } + ], + "cudaVersion": "11.8" +} \ No newline at end of file diff --git a/wandb/run-20241101_200535-lnp8ii96/logs/debug-internal.log b/wandb/run-20241101_200535-lnp8ii96/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..b15c7e8284a471f716f206a2b2b1adbb8688b05c --- /dev/null +++ b/wandb/run-20241101_200535-lnp8ii96/logs/debug-internal.log @@ -0,0 +1,8 @@ +{"time":"2024-11-01T20:05:35.626320342-04:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-11-01T20:05:35.626341822-04:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_200535-lnp8ii96/logs/debug-core.log"} +{"time":"2024-11-01T20:05:35.736138328-04:00","level":"INFO","msg":"created new stream","id":"lnp8ii96"} +{"time":"2024-11-01T20:05:35.736185149-04:00","level":"INFO","msg":"stream: started","id":"lnp8ii96"} +{"time":"2024-11-01T20:05:35.736261459-04:00","level":"INFO","msg":"sender: started","stream_id":"lnp8ii96"} +{"time":"2024-11-01T20:05:35.736245139-04:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"lnp8ii96"}} +{"time":"2024-11-01T20:05:35.73631463-04:00","level":"INFO","msg":"handler: started","stream_id":{"value":"lnp8ii96"}} +{"time":"2024-11-01T20:05:35.90118916-04:00","level":"INFO","msg":"Starting system monitor"} diff --git a/wandb/run-20241101_200535-lnp8ii96/logs/debug.log b/wandb/run-20241101_200535-lnp8ii96/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..c8ed2c7bc892a9dc71fbe6870154794c0f74095a --- /dev/null +++ b/wandb/run-20241101_200535-lnp8ii96/logs/debug.log @@ -0,0 +1,26 @@ +2024-11-01 20:05:35,620 INFO MainThread:871229 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-11-01 20:05:35,620 INFO MainThread:871229 [wandb_setup.py:_flush():79] Configure stats pid to 871229 +2024-11-01 20:05:35,620 INFO MainThread:871229 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-11-01 20:05:35,620 INFO MainThread:871229 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-11-01 20:05:35,620 INFO MainThread:871229 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-01 20:05:35,620 INFO MainThread:871229 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-01 20:05:35,620 INFO MainThread:871229 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'} +2024-11-01 20:05:35,620 INFO MainThread:871229 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-01 20:05:35,620 INFO MainThread:871229 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_200535-lnp8ii96/logs/debug.log +2024-11-01 20:05:35,620 INFO MainThread:871229 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241101_200535-lnp8ii96/logs/debug-internal.log +2024-11-01 20:05:35,621 INFO MainThread:871229 [wandb_init.py:init():621] calling init triggers +2024-11-01 20:05:35,621 INFO MainThread:871229 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-11-01 20:05:35,621 INFO MainThread:871229 [wandb_init.py:init():671] starting backend +2024-11-01 20:05:35,621 INFO MainThread:871229 [wandb_init.py:init():675] sending inform_init request +2024-11-01 20:05:35,622 INFO MainThread:871229 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-01 20:05:35,623 INFO MainThread:871229 [wandb_init.py:init():688] backend started and connected +2024-11-01 20:05:35,626 INFO MainThread:871229 [wandb_init.py:init():783] updated telemetry +2024-11-01 20:05:35,646 INFO MainThread:871229 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-11-01 20:05:35,897 INFO MainThread:871229 [wandb_init.py:init():867] starting run threads in backend +2024-11-01 20:05:35,987 INFO MainThread:871229 [wandb_run.py:_console_start():2463] atexit reg +2024-11-01 20:05:35,987 INFO MainThread:871229 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-11-01 20:05:35,987 INFO MainThread:871229 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-11-01 20:05:35,987 INFO MainThread:871229 [wandb_run.py:_redirect():2401] Redirects installed. +2024-11-01 20:05:35,989 INFO MainThread:871229 [wandb_init.py:init():911] run started, returning control to user process +2024-11-01 20:05:35,989 INFO MainThread:871229 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_nondeterministic', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0, 'lr': 5e-06} diff --git a/wandb/run-20241101_200535-xloij0da/run-xloij0da.wandb b/wandb/run-20241101_200535-xloij0da/run-xloij0da.wandb new file mode 100644 index 0000000000000000000000000000000000000000..0d7f6b5ff67563b7a9a708bb03e54cbe17b3efe6 --- /dev/null +++ b/wandb/run-20241101_200535-xloij0da/run-xloij0da.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86f830d3dae48b66e8e3e2ca7fdbd7abc67f7ca270bb41d81aa5d822bd8fc2de +size 131072 diff --git a/wandb/run-20241105_155905-adxztc74/files/config.yaml b/wandb/run-20241105_155905-adxztc74/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba0e0eb5aa81d7186d91b3c8f342ad5574a4c100 --- /dev/null +++ b/wandb/run-20241105_155905-adxztc74/files/config.yaml @@ -0,0 +1,49 @@ +_wandb: + value: + cli_version: 0.18.5 + m: [] + python_version: 3.9.19 + t: + "1": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "2": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "3": + - 13 + - 23 + - 55 + "4": 3.9.19 + "5": 0.18.5 + "6": 4.45.1 + "8": + - 5 + "12": 0.18.5 + "13": linux-x86_64 +batch_size: + value: 3 +epoch: + value: 3 +lr: + value: 5e-06 +perturbation: + value: shuffle_deterministic21 +seed: + value: 0 +train_set: + value: 10M diff --git a/wandb/run-20241105_155905-adxztc74/files/output.log b/wandb/run-20241105_155905-adxztc74/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..4889313e9210ddc0bf0e53a0b8deb84b34f76597 --- /dev/null +++ b/wandb/run-20241105_155905-adxztc74/files/output.log @@ -0,0 +1,19 @@ +Traceback (most recent call last): + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/pathlib.py", line 1323, in mkdir + self._accessor.mkdir(self, mode) +FileNotFoundError: [Errno 2] No such file or directory: '/home/chunhui/.cache/huggingface/datasets/babylm_dataset_test/babylm_shuffle_deterministic21_10M_seed0/0.0.0' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 165, in + dataset = load_dataset('babylm_dataset_test.py', name=dataset_name, trust_remote_code=True) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/load.py", line 2096, in load_dataset + builder_instance.download_and_prepare( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/builder.py", line 855, in download_and_prepare + Path(self._output_dir).parent.mkdir(parents=True, exist_ok=True) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/pathlib.py", line 1327, in mkdir + self.parent.mkdir(parents=True, exist_ok=True) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/pathlib.py", line 1323, in mkdir + self._accessor.mkdir(self, mode) +OSError: [Errno 28] No space left on device: '/home/chunhui/.cache/huggingface/datasets/babylm_dataset_test/babylm_shuffle_deterministic21_10M_seed0' diff --git a/wandb/run-20241105_155905-adxztc74/files/requirements.txt b/wandb/run-20241105_155905-adxztc74/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..95a931302e269cc9e4fa5b719b6511f176ee2416 --- /dev/null +++ b/wandb/run-20241105_155905-adxztc74/files/requirements.txt @@ -0,0 +1,147 @@ +funcsigs==1.0.2 +sentry-sdk==2.17.0 +multiprocess==0.70.16 +numpy==1.26.2 +pluralizer==1.2.0 +debugpy==1.6.7 +nvidia-cudnn-cu11==8.5.0.96 +deepspeed==0.15.2 +data==0.4 +pandas==2.1.3 +tomli==2.0.1 +charset-normalizer==3.3.2 +attrs==24.2.0 +aiosignal==1.3.1 +fsspec==2023.10.0 +nvidia-cusparse-cu11==11.7.4.91 +zipp==3.12.0 +mypy-extensions==1.0.0 +datasets==3.0.1 +joblib==1.3.2 +hjson==3.1.0 +traitlets==5.7.1 +stack-data==0.6.0 +transformers==4.45.1 +sympy==1.11.1 +Pygments==2.15.0 +docker-pycreds==0.4.0 +dill==0.3.8 +wheel==0.44.0 +prompt-toolkit==3.0.30 +parso==0.8.3 +ipykernel==6.23.1 +pyarrow==17.0.0 +certifi==2023.11.17 +nvidia-cufft-cu11==10.9.0.58 +six==1.16.0 +pydantic==2.9.2 +click==8.1.7 +nest-asyncio==1.5.6 +gmpy2==2.1.0 +matplotlib==3.8.2 +scipy==1.11.4 +typing_extensions==4.12.2 +statsmodels==0.14.0 +huggingface-hub==0.25.0 +frozenlist==1.4.1 +gpustat==1.1.1 +nvidia-nvtx-cu11==11.7.91 +safetensors==0.4.5 +stanza==1.9.2 +decorator==5.1.1 +seaborn==0.13.0 +sentencepiece==0.2.0 +PyYAML==6.0.1 +black==24.8.0 +protobuf==4.25.1 +pickleshare==0.7.5 +peft==0.13.0 +triton==2.0.0 +nvidia-cuda-runtime-cu11==11.7.99 +Jinja2==3.1.2 +nvidia-cusolver-cu11==11.4.0.1 +executing==1.2.0 +jupyter_client==8.1.0 +pluggy==1.3.0 +cmake==3.30.3 +pytz==2023.3.post1 +aiohappyeyeballs==2.4.2 +kiwisolver==1.4.5 +py-cpuinfo==9.0.0 +Pillow==10.1.0 +ptyprocess==0.7.0 +importlib_resources==6.4.5 +GitPython==3.1.43 +importlib-metadata==6.0.0 +iniconfig==2.0.0 +scikit-learn==1.3.2 +exceptiongroup==1.1.0 +networkx==2.8.6 +accelerate==1.0.0 +nltk==3.8.1 +shutilwhich==1.1.0 +fonttools==4.45.1 +future==0.18.3 +aiohttp==3.10.6 +wcwidth==0.2.5 +idna==3.6 +filelock==3.12.2 +pathspec==0.12.1 +jupyter_core==5.1.0 +lit==18.1.8 +nvidia-curand-cu11==10.2.10.91 +nvidia-cublas-cu11==11.10.3.66 +nvidia-ml-py==12.560.30 +msgpack==1.1.0 +python-dateutil==2.8.2 +blessed==1.20.0 +packaging==23.0 +gitdb==4.0.11 +yarl==1.13.0 +emoji==2.8.0 +tzdata==2023.3 +cycler==0.12.1 +tornado==6.2 +backcall==0.2.0 +plotnine==0.12.4 +ninja==1.11.1.1 +latex==0.7.0 +wandb==0.18.5 +setproctitle==1.3.3 +threadpoolctl==3.2.0 +requests==2.32.3 +pyparsing==3.1.1 +smmap==5.0.1 +pyzmq==23.0.0 +async-timeout==4.0.3 +annotated-types==0.7.0 +matplotlib-inline==0.1.6 +latexcodec==1.0.0 +ipython==8.0.0 +patsy==0.5.3 +contourpy==1.2.0 +multidict==6.1.0 +mizani==0.9.3 +urllib3==2.1.0 +tokenizers==0.20.0 +MarkupSafe==2.1.2 +pip==24.2 +pexpect==4.8.0 +tqdm==4.66.5 +jedi==0.18.2 +pydantic_core==2.23.4 +tempdir==0.7.1 +mpmath==1.2.1 +setuptools==72.1.0 +pytest==7.4.3 +pure-eval==0.2.2 +psutil==5.9.1 +comm==0.1.2 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +regex==2023.10.3 +platformdirs==2.5.2 +asttokens==2.2.1 +torch==2.0.0 +nvidia-nccl-cu11==2.14.3 +xxhash==3.5.0 diff --git a/wandb/run-20241105_155905-adxztc74/files/wandb-metadata.json b/wandb/run-20241105_155905-adxztc74/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..0aaa6746d64adfd6700257736f54b07ac1ddb342 --- /dev/null +++ b/wandb/run-20241105_155905-adxztc74/files/wandb-metadata.json @@ -0,0 +1,44 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-11-05T20:59:05.914825Z", + "args": [ + "--perturbation", + "shuffle_deterministic21", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "3", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", + "codePath": "train/train_deep_wandb.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_deep_wandb.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "disk": { + "/": { + "total": "1888559353856", + "used": "1792550322176" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + } +} \ No newline at end of file diff --git a/wandb/run-20241105_155905-adxztc74/files/wandb-summary.json b/wandb/run-20241105_155905-adxztc74/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..779a0b5a17ed7980591fb713a2a4db505a51ed4b --- /dev/null +++ b/wandb/run-20241105_155905-adxztc74/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":8}} \ No newline at end of file diff --git a/wandb/run-20241105_155905-adxztc74/logs/debug-internal.log b/wandb/run-20241105_155905-adxztc74/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..634f7118b1e672be037a8a05b4b5ad4b57b798ad --- /dev/null +++ b/wandb/run-20241105_155905-adxztc74/logs/debug-internal.log @@ -0,0 +1,18 @@ +{"time":"2024-11-05T15:59:05.91775728-05:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-11-05T15:59:05.91777683-05:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_155905-adxztc74/logs/debug-core.log"} +{"time":"2024-11-05T15:59:10.952154954-05:00","level":"INFO","msg":"created new stream","id":"adxztc74"} +{"time":"2024-11-05T15:59:10.952196724-05:00","level":"INFO","msg":"stream: started","id":"adxztc74"} +{"time":"2024-11-05T15:59:10.952239154-05:00","level":"INFO","msg":"sender: started","stream_id":"adxztc74"} +{"time":"2024-11-05T15:59:10.952206264-05:00","level":"INFO","msg":"handler: started","stream_id":{"value":"adxztc74"}} +{"time":"2024-11-05T15:59:10.952238964-05:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"adxztc74"}} +{"time":"2024-11-05T15:59:11.536374693-05:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/graphql"} +{"time":"2024-11-05T15:59:14.076098382-05:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-11-05T15:59:14.191332644-05:00","level":"INFO","msg":"stream: closing","id":"adxztc74"} +{"time":"2024-11-05T15:59:14.191368314-05:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2024-11-05T15:59:14.191421524-05:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2024-11-05T15:59:14.461833728-05:00","level":"ERROR","msg":"sender: sendDefer: failed to build job artifact","error":"failed to write data to file: write /tmp/tmpfile-240625076: no space left on device"} +{"time":"2024-11-05T15:59:14.724659519-05:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2024-11-05T15:59:14.905874368-05:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"adxztc74"}} +{"time":"2024-11-05T15:59:14.905929738-05:00","level":"INFO","msg":"sender: closed","stream_id":"adxztc74"} +{"time":"2024-11-05T15:59:14.905921668-05:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"adxztc74"}} +{"time":"2024-11-05T15:59:14.906009548-05:00","level":"INFO","msg":"stream: closed","id":"adxztc74"} diff --git a/wandb/run-20241105_155905-adxztc74/logs/debug.log b/wandb/run-20241105_155905-adxztc74/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..931a292f4f45ea923a282ccfddfffb69aa39f693 --- /dev/null +++ b/wandb/run-20241105_155905-adxztc74/logs/debug.log @@ -0,0 +1,27 @@ +2024-11-05 15:59:05,913 INFO MainThread:1768668 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-11-05 15:59:05,913 INFO MainThread:1768668 [wandb_setup.py:_flush():79] Configure stats pid to 1768668 +2024-11-05 15:59:05,913 INFO MainThread:1768668 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-11-05 15:59:05,913 INFO MainThread:1768668 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-11-05 15:59:05,913 INFO MainThread:1768668 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-05 15:59:05,913 INFO MainThread:1768668 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-05 15:59:05,913 INFO MainThread:1768668 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'} +2024-11-05 15:59:05,913 INFO MainThread:1768668 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-05 15:59:05,913 INFO MainThread:1768668 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_155905-adxztc74/logs/debug.log +2024-11-05 15:59:05,913 INFO MainThread:1768668 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_155905-adxztc74/logs/debug-internal.log +2024-11-05 15:59:05,913 INFO MainThread:1768668 [wandb_init.py:init():621] calling init triggers +2024-11-05 15:59:05,913 INFO MainThread:1768668 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-11-05 15:59:05,913 INFO MainThread:1768668 [wandb_init.py:init():671] starting backend +2024-11-05 15:59:05,913 INFO MainThread:1768668 [wandb_init.py:init():675] sending inform_init request +2024-11-05 15:59:05,914 INFO MainThread:1768668 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-05 15:59:05,914 INFO MainThread:1768668 [wandb_init.py:init():688] backend started and connected +2024-11-05 15:59:05,917 INFO MainThread:1768668 [wandb_init.py:init():783] updated telemetry +2024-11-05 15:59:05,944 INFO MainThread:1768668 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-11-05 15:59:14,072 INFO MainThread:1768668 [wandb_init.py:init():867] starting run threads in backend +2024-11-05 15:59:14,167 INFO MainThread:1768668 [wandb_run.py:_console_start():2463] atexit reg +2024-11-05 15:59:14,167 INFO MainThread:1768668 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-11-05 15:59:14,167 INFO MainThread:1768668 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-11-05 15:59:14,167 INFO MainThread:1768668 [wandb_run.py:_redirect():2401] Redirects installed. +2024-11-05 15:59:14,168 INFO MainThread:1768668 [wandb_init.py:init():911] run started, returning control to user process +2024-11-05 15:59:14,169 INFO MainThread:1768668 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_deterministic21', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0, 'lr': 5e-06} +2024-11-05 15:59:14,191 WARNING MsgRouterThr:1768668 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20241105_155905-adxztc74/run-adxztc74.wandb b/wandb/run-20241105_155905-adxztc74/run-adxztc74.wandb new file mode 100644 index 0000000000000000000000000000000000000000..03f56817684823cc68768cf0cd21d6fe6705ed16 Binary files /dev/null and b/wandb/run-20241105_155905-adxztc74/run-adxztc74.wandb differ diff --git a/wandb/run-20241105_155954-daaq0lj0/files/config.yaml b/wandb/run-20241105_155954-daaq0lj0/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba0e0eb5aa81d7186d91b3c8f342ad5574a4c100 --- /dev/null +++ b/wandb/run-20241105_155954-daaq0lj0/files/config.yaml @@ -0,0 +1,49 @@ +_wandb: + value: + cli_version: 0.18.5 + m: [] + python_version: 3.9.19 + t: + "1": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "2": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "3": + - 13 + - 23 + - 55 + "4": 3.9.19 + "5": 0.18.5 + "6": 4.45.1 + "8": + - 5 + "12": 0.18.5 + "13": linux-x86_64 +batch_size: + value: 3 +epoch: + value: 3 +lr: + value: 5e-06 +perturbation: + value: shuffle_deterministic21 +seed: + value: 0 +train_set: + value: 10M diff --git a/wandb/run-20241105_155954-daaq0lj0/files/output.log b/wandb/run-20241105_155954-daaq0lj0/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..4889313e9210ddc0bf0e53a0b8deb84b34f76597 --- /dev/null +++ b/wandb/run-20241105_155954-daaq0lj0/files/output.log @@ -0,0 +1,19 @@ +Traceback (most recent call last): + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/pathlib.py", line 1323, in mkdir + self._accessor.mkdir(self, mode) +FileNotFoundError: [Errno 2] No such file or directory: '/home/chunhui/.cache/huggingface/datasets/babylm_dataset_test/babylm_shuffle_deterministic21_10M_seed0/0.0.0' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", line 165, in + dataset = load_dataset('babylm_dataset_test.py', name=dataset_name, trust_remote_code=True) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/load.py", line 2096, in load_dataset + builder_instance.download_and_prepare( + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/datasets/builder.py", line 855, in download_and_prepare + Path(self._output_dir).parent.mkdir(parents=True, exist_ok=True) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/pathlib.py", line 1327, in mkdir + self.parent.mkdir(parents=True, exist_ok=True) + File "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/pathlib.py", line 1323, in mkdir + self._accessor.mkdir(self, mode) +OSError: [Errno 28] No space left on device: '/home/chunhui/.cache/huggingface/datasets/babylm_dataset_test/babylm_shuffle_deterministic21_10M_seed0' diff --git a/wandb/run-20241105_155954-daaq0lj0/files/requirements.txt b/wandb/run-20241105_155954-daaq0lj0/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..95a931302e269cc9e4fa5b719b6511f176ee2416 --- /dev/null +++ b/wandb/run-20241105_155954-daaq0lj0/files/requirements.txt @@ -0,0 +1,147 @@ +funcsigs==1.0.2 +sentry-sdk==2.17.0 +multiprocess==0.70.16 +numpy==1.26.2 +pluralizer==1.2.0 +debugpy==1.6.7 +nvidia-cudnn-cu11==8.5.0.96 +deepspeed==0.15.2 +data==0.4 +pandas==2.1.3 +tomli==2.0.1 +charset-normalizer==3.3.2 +attrs==24.2.0 +aiosignal==1.3.1 +fsspec==2023.10.0 +nvidia-cusparse-cu11==11.7.4.91 +zipp==3.12.0 +mypy-extensions==1.0.0 +datasets==3.0.1 +joblib==1.3.2 +hjson==3.1.0 +traitlets==5.7.1 +stack-data==0.6.0 +transformers==4.45.1 +sympy==1.11.1 +Pygments==2.15.0 +docker-pycreds==0.4.0 +dill==0.3.8 +wheel==0.44.0 +prompt-toolkit==3.0.30 +parso==0.8.3 +ipykernel==6.23.1 +pyarrow==17.0.0 +certifi==2023.11.17 +nvidia-cufft-cu11==10.9.0.58 +six==1.16.0 +pydantic==2.9.2 +click==8.1.7 +nest-asyncio==1.5.6 +gmpy2==2.1.0 +matplotlib==3.8.2 +scipy==1.11.4 +typing_extensions==4.12.2 +statsmodels==0.14.0 +huggingface-hub==0.25.0 +frozenlist==1.4.1 +gpustat==1.1.1 +nvidia-nvtx-cu11==11.7.91 +safetensors==0.4.5 +stanza==1.9.2 +decorator==5.1.1 +seaborn==0.13.0 +sentencepiece==0.2.0 +PyYAML==6.0.1 +black==24.8.0 +protobuf==4.25.1 +pickleshare==0.7.5 +peft==0.13.0 +triton==2.0.0 +nvidia-cuda-runtime-cu11==11.7.99 +Jinja2==3.1.2 +nvidia-cusolver-cu11==11.4.0.1 +executing==1.2.0 +jupyter_client==8.1.0 +pluggy==1.3.0 +cmake==3.30.3 +pytz==2023.3.post1 +aiohappyeyeballs==2.4.2 +kiwisolver==1.4.5 +py-cpuinfo==9.0.0 +Pillow==10.1.0 +ptyprocess==0.7.0 +importlib_resources==6.4.5 +GitPython==3.1.43 +importlib-metadata==6.0.0 +iniconfig==2.0.0 +scikit-learn==1.3.2 +exceptiongroup==1.1.0 +networkx==2.8.6 +accelerate==1.0.0 +nltk==3.8.1 +shutilwhich==1.1.0 +fonttools==4.45.1 +future==0.18.3 +aiohttp==3.10.6 +wcwidth==0.2.5 +idna==3.6 +filelock==3.12.2 +pathspec==0.12.1 +jupyter_core==5.1.0 +lit==18.1.8 +nvidia-curand-cu11==10.2.10.91 +nvidia-cublas-cu11==11.10.3.66 +nvidia-ml-py==12.560.30 +msgpack==1.1.0 +python-dateutil==2.8.2 +blessed==1.20.0 +packaging==23.0 +gitdb==4.0.11 +yarl==1.13.0 +emoji==2.8.0 +tzdata==2023.3 +cycler==0.12.1 +tornado==6.2 +backcall==0.2.0 +plotnine==0.12.4 +ninja==1.11.1.1 +latex==0.7.0 +wandb==0.18.5 +setproctitle==1.3.3 +threadpoolctl==3.2.0 +requests==2.32.3 +pyparsing==3.1.1 +smmap==5.0.1 +pyzmq==23.0.0 +async-timeout==4.0.3 +annotated-types==0.7.0 +matplotlib-inline==0.1.6 +latexcodec==1.0.0 +ipython==8.0.0 +patsy==0.5.3 +contourpy==1.2.0 +multidict==6.1.0 +mizani==0.9.3 +urllib3==2.1.0 +tokenizers==0.20.0 +MarkupSafe==2.1.2 +pip==24.2 +pexpect==4.8.0 +tqdm==4.66.5 +jedi==0.18.2 +pydantic_core==2.23.4 +tempdir==0.7.1 +mpmath==1.2.1 +setuptools==72.1.0 +pytest==7.4.3 +pure-eval==0.2.2 +psutil==5.9.1 +comm==0.1.2 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +regex==2023.10.3 +platformdirs==2.5.2 +asttokens==2.2.1 +torch==2.0.0 +nvidia-nccl-cu11==2.14.3 +xxhash==3.5.0 diff --git a/wandb/run-20241105_155954-daaq0lj0/files/wandb-metadata.json b/wandb/run-20241105_155954-daaq0lj0/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..64f0564573588da1e58121d1eb45b087ad523a3d --- /dev/null +++ b/wandb/run-20241105_155954-daaq0lj0/files/wandb-metadata.json @@ -0,0 +1,44 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-11-05T20:59:54.226453Z", + "args": [ + "--perturbation", + "shuffle_deterministic21", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "3", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py", + "codePath": "train/train_deep_wandb.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_deep_wandb.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "disk": { + "/": { + "total": "1888559353856", + "used": "1792550322176" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + } +} \ No newline at end of file diff --git a/wandb/run-20241105_155954-daaq0lj0/files/wandb-summary.json b/wandb/run-20241105_155954-daaq0lj0/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..a59211b910c7b68e6827eb6c887d30d98244727c --- /dev/null +++ b/wandb/run-20241105_155954-daaq0lj0/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":5}} \ No newline at end of file diff --git a/wandb/run-20241105_155954-daaq0lj0/logs/debug-internal.log b/wandb/run-20241105_155954-daaq0lj0/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..9ed0e5ce9772c50fff5100042ee7a08075798a71 --- /dev/null +++ b/wandb/run-20241105_155954-daaq0lj0/logs/debug-internal.log @@ -0,0 +1,17 @@ +{"time":"2024-11-05T15:59:54.228340071-05:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-11-05T15:59:54.228350531-05:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_155954-daaq0lj0/logs/debug-core.log"} +{"time":"2024-11-05T15:59:59.257106907-05:00","level":"INFO","msg":"created new stream","id":"daaq0lj0"} +{"time":"2024-11-05T15:59:59.257158837-05:00","level":"INFO","msg":"stream: started","id":"daaq0lj0"} +{"time":"2024-11-05T15:59:59.257192978-05:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"daaq0lj0"}} +{"time":"2024-11-05T15:59:59.257266618-05:00","level":"INFO","msg":"handler: started","stream_id":{"value":"daaq0lj0"}} +{"time":"2024-11-05T15:59:59.257319508-05:00","level":"INFO","msg":"sender: started","stream_id":"daaq0lj0"} +{"time":"2024-11-05T15:59:59.479513375-05:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-11-05T15:59:59.586100372-05:00","level":"INFO","msg":"stream: closing","id":"daaq0lj0"} +{"time":"2024-11-05T15:59:59.586134512-05:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2024-11-05T15:59:59.586189483-05:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2024-11-05T15:59:59.853662105-05:00","level":"ERROR","msg":"sender: sendDefer: failed to build job artifact","error":"failed to write data to file: write /tmp/tmpfile-1980955257: no space left on device"} +{"time":"2024-11-05T16:00:00.124940441-05:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2024-11-05T16:00:00.239093907-05:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"daaq0lj0"}} +{"time":"2024-11-05T16:00:00.239133727-05:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"daaq0lj0"}} +{"time":"2024-11-05T16:00:00.239161847-05:00","level":"INFO","msg":"sender: closed","stream_id":"daaq0lj0"} +{"time":"2024-11-05T16:00:00.239197537-05:00","level":"INFO","msg":"stream: closed","id":"daaq0lj0"} diff --git a/wandb/run-20241105_155954-daaq0lj0/run-daaq0lj0.wandb b/wandb/run-20241105_155954-daaq0lj0/run-daaq0lj0.wandb new file mode 100644 index 0000000000000000000000000000000000000000..d9bb3e0a3b9703a0283e38de7d0b3a6c61a811bd Binary files /dev/null and b/wandb/run-20241105_155954-daaq0lj0/run-daaq0lj0.wandb differ diff --git a/wandb/run-20241105_161832-j5bvgnk0/files/output.log b/wandb/run-20241105_161832-j5bvgnk0/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..672c7087ee59fafbe4e72e0b4fa11ca0be60bdfc --- /dev/null +++ b/wandb/run-20241105_161832-j5bvgnk0/files/output.log @@ -0,0 +1,20 @@ +model.safetensors.index.json: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████| 20.9k/20.9k [00:00<00:00, 5.82MB/s] +model-00001-of-00002.safetensors: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████| 4.97G/4.97G [01:58<00:00, 42.0MB/s] +model-00002-of-00002.safetensors: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████| 1.46G/1.46G [00:34<00:00, 42.4MB/s] +Downloading shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [02:33<00:00, 76.50s/it] +Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:09<00:00, 4.68s/it] +generation_config.json: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 185/185 [00:00<00:00, 71.6kB/s] +Map: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 16425/16425 [00:44<00:00, 370.09 examples/s] +Map: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 17013/17013 [00:45<00:00, 376.64 examples/s] +tokenized_valid: Dataset({ + features: ['input_ids', 'attention_mask'], + num_rows: 1000 +}) +/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +[2024-11-05 16:23:09,770] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-11-05 16:23:20,090] [INFO] [comm.py:652:init_distributed] cdb=None +Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination +Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root... +Loading extension module cpu_adam... +Time to load cpu_adam op: 31.75210690498352 seconds diff --git a/wandb/run-20241105_161832-j5bvgnk0/logs/debug-internal.log b/wandb/run-20241105_161832-j5bvgnk0/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..c4c5e14ebb4c1bdb26a50e781f57d548483abe58 --- /dev/null +++ b/wandb/run-20241105_161832-j5bvgnk0/logs/debug-internal.log @@ -0,0 +1,8 @@ +{"time":"2024-11-05T16:18:32.758390242-05:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-11-05T16:18:32.758400202-05:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_161832-j5bvgnk0/logs/debug-core.log"} +{"time":"2024-11-05T16:18:32.864764472-05:00","level":"INFO","msg":"created new stream","id":"j5bvgnk0"} +{"time":"2024-11-05T16:18:32.864793412-05:00","level":"INFO","msg":"stream: started","id":"j5bvgnk0"} +{"time":"2024-11-05T16:18:32.864850762-05:00","level":"INFO","msg":"sender: started","stream_id":"j5bvgnk0"} +{"time":"2024-11-05T16:18:32.864822162-05:00","level":"INFO","msg":"handler: started","stream_id":{"value":"j5bvgnk0"}} +{"time":"2024-11-05T16:18:32.864818442-05:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"j5bvgnk0"}} +{"time":"2024-11-05T16:18:33.077866324-05:00","level":"INFO","msg":"Starting system monitor"} diff --git a/wandb/run-20241105_161832-j5bvgnk0/logs/debug.log b/wandb/run-20241105_161832-j5bvgnk0/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..d198c0f571d2eab2b6b54af0a0e0560ba30f69f2 --- /dev/null +++ b/wandb/run-20241105_161832-j5bvgnk0/logs/debug.log @@ -0,0 +1,26 @@ +2024-11-05 16:18:32,753 INFO MainThread:1773598 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-11-05 16:18:32,753 INFO MainThread:1773598 [wandb_setup.py:_flush():79] Configure stats pid to 1773598 +2024-11-05 16:18:32,753 INFO MainThread:1773598 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-11-05 16:18:32,753 INFO MainThread:1773598 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-11-05 16:18:32,753 INFO MainThread:1773598 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-05 16:18:32,753 INFO MainThread:1773598 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-05 16:18:32,753 INFO MainThread:1773598 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_deep_wandb.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_deep_wandb.py'} +2024-11-05 16:18:32,753 INFO MainThread:1773598 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-05 16:18:32,754 INFO MainThread:1773598 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_161832-j5bvgnk0/logs/debug.log +2024-11-05 16:18:32,754 INFO MainThread:1773598 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241105_161832-j5bvgnk0/logs/debug-internal.log +2024-11-05 16:18:32,754 INFO MainThread:1773598 [wandb_init.py:init():621] calling init triggers +2024-11-05 16:18:32,754 INFO MainThread:1773598 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-11-05 16:18:32,754 INFO MainThread:1773598 [wandb_init.py:init():671] starting backend +2024-11-05 16:18:32,754 INFO MainThread:1773598 [wandb_init.py:init():675] sending inform_init request +2024-11-05 16:18:32,755 INFO MainThread:1773598 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-05 16:18:32,756 INFO MainThread:1773598 [wandb_init.py:init():688] backend started and connected +2024-11-05 16:18:32,759 INFO MainThread:1773598 [wandb_init.py:init():783] updated telemetry +2024-11-05 16:18:32,779 INFO MainThread:1773598 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-11-05 16:18:33,075 INFO MainThread:1773598 [wandb_init.py:init():867] starting run threads in backend +2024-11-05 16:18:33,162 INFO MainThread:1773598 [wandb_run.py:_console_start():2463] atexit reg +2024-11-05 16:18:33,162 INFO MainThread:1773598 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-11-05 16:18:33,162 INFO MainThread:1773598 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-11-05 16:18:33,162 INFO MainThread:1773598 [wandb_run.py:_redirect():2401] Redirects installed. +2024-11-05 16:18:33,164 INFO MainThread:1773598 [wandb_init.py:init():911] run started, returning control to user process +2024-11-05 16:18:33,164 INFO MainThread:1773598 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_deterministic21', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0, 'lr': 5e-06} diff --git a/wandb/run-20241115_125218-bbgu27nw/files/requirements.txt b/wandb/run-20241115_125218-bbgu27nw/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..95a931302e269cc9e4fa5b719b6511f176ee2416 --- /dev/null +++ b/wandb/run-20241115_125218-bbgu27nw/files/requirements.txt @@ -0,0 +1,147 @@ +funcsigs==1.0.2 +sentry-sdk==2.17.0 +multiprocess==0.70.16 +numpy==1.26.2 +pluralizer==1.2.0 +debugpy==1.6.7 +nvidia-cudnn-cu11==8.5.0.96 +deepspeed==0.15.2 +data==0.4 +pandas==2.1.3 +tomli==2.0.1 +charset-normalizer==3.3.2 +attrs==24.2.0 +aiosignal==1.3.1 +fsspec==2023.10.0 +nvidia-cusparse-cu11==11.7.4.91 +zipp==3.12.0 +mypy-extensions==1.0.0 +datasets==3.0.1 +joblib==1.3.2 +hjson==3.1.0 +traitlets==5.7.1 +stack-data==0.6.0 +transformers==4.45.1 +sympy==1.11.1 +Pygments==2.15.0 +docker-pycreds==0.4.0 +dill==0.3.8 +wheel==0.44.0 +prompt-toolkit==3.0.30 +parso==0.8.3 +ipykernel==6.23.1 +pyarrow==17.0.0 +certifi==2023.11.17 +nvidia-cufft-cu11==10.9.0.58 +six==1.16.0 +pydantic==2.9.2 +click==8.1.7 +nest-asyncio==1.5.6 +gmpy2==2.1.0 +matplotlib==3.8.2 +scipy==1.11.4 +typing_extensions==4.12.2 +statsmodels==0.14.0 +huggingface-hub==0.25.0 +frozenlist==1.4.1 +gpustat==1.1.1 +nvidia-nvtx-cu11==11.7.91 +safetensors==0.4.5 +stanza==1.9.2 +decorator==5.1.1 +seaborn==0.13.0 +sentencepiece==0.2.0 +PyYAML==6.0.1 +black==24.8.0 +protobuf==4.25.1 +pickleshare==0.7.5 +peft==0.13.0 +triton==2.0.0 +nvidia-cuda-runtime-cu11==11.7.99 +Jinja2==3.1.2 +nvidia-cusolver-cu11==11.4.0.1 +executing==1.2.0 +jupyter_client==8.1.0 +pluggy==1.3.0 +cmake==3.30.3 +pytz==2023.3.post1 +aiohappyeyeballs==2.4.2 +kiwisolver==1.4.5 +py-cpuinfo==9.0.0 +Pillow==10.1.0 +ptyprocess==0.7.0 +importlib_resources==6.4.5 +GitPython==3.1.43 +importlib-metadata==6.0.0 +iniconfig==2.0.0 +scikit-learn==1.3.2 +exceptiongroup==1.1.0 +networkx==2.8.6 +accelerate==1.0.0 +nltk==3.8.1 +shutilwhich==1.1.0 +fonttools==4.45.1 +future==0.18.3 +aiohttp==3.10.6 +wcwidth==0.2.5 +idna==3.6 +filelock==3.12.2 +pathspec==0.12.1 +jupyter_core==5.1.0 +lit==18.1.8 +nvidia-curand-cu11==10.2.10.91 +nvidia-cublas-cu11==11.10.3.66 +nvidia-ml-py==12.560.30 +msgpack==1.1.0 +python-dateutil==2.8.2 +blessed==1.20.0 +packaging==23.0 +gitdb==4.0.11 +yarl==1.13.0 +emoji==2.8.0 +tzdata==2023.3 +cycler==0.12.1 +tornado==6.2 +backcall==0.2.0 +plotnine==0.12.4 +ninja==1.11.1.1 +latex==0.7.0 +wandb==0.18.5 +setproctitle==1.3.3 +threadpoolctl==3.2.0 +requests==2.32.3 +pyparsing==3.1.1 +smmap==5.0.1 +pyzmq==23.0.0 +async-timeout==4.0.3 +annotated-types==0.7.0 +matplotlib-inline==0.1.6 +latexcodec==1.0.0 +ipython==8.0.0 +patsy==0.5.3 +contourpy==1.2.0 +multidict==6.1.0 +mizani==0.9.3 +urllib3==2.1.0 +tokenizers==0.20.0 +MarkupSafe==2.1.2 +pip==24.2 +pexpect==4.8.0 +tqdm==4.66.5 +jedi==0.18.2 +pydantic_core==2.23.4 +tempdir==0.7.1 +mpmath==1.2.1 +setuptools==72.1.0 +pytest==7.4.3 +pure-eval==0.2.2 +psutil==5.9.1 +comm==0.1.2 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +regex==2023.10.3 +platformdirs==2.5.2 +asttokens==2.2.1 +torch==2.0.0 +nvidia-nccl-cu11==2.14.3 +xxhash==3.5.0 diff --git a/wandb/run-20241115_125218-bbgu27nw/logs/debug-internal.log b/wandb/run-20241115_125218-bbgu27nw/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..08bc969f73adadda2aff2238ac32f290c20533f4 --- /dev/null +++ b/wandb/run-20241115_125218-bbgu27nw/logs/debug-internal.log @@ -0,0 +1,8 @@ +{"time":"2024-11-15T12:52:18.160385832-05:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-11-15T12:52:18.160416542-05:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241115_125218-bbgu27nw/logs/debug-core.log"} +{"time":"2024-11-15T12:52:18.272320302-05:00","level":"INFO","msg":"created new stream","id":"bbgu27nw"} +{"time":"2024-11-15T12:52:18.272387392-05:00","level":"INFO","msg":"stream: started","id":"bbgu27nw"} +{"time":"2024-11-15T12:52:18.272417092-05:00","level":"INFO","msg":"sender: started","stream_id":"bbgu27nw"} +{"time":"2024-11-15T12:52:18.272408542-05:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"bbgu27nw"}} +{"time":"2024-11-15T12:52:18.272421752-05:00","level":"INFO","msg":"handler: started","stream_id":{"value":"bbgu27nw"}} +{"time":"2024-11-15T12:52:18.500078814-05:00","level":"INFO","msg":"Starting system monitor"} diff --git a/wandb/run-20241115_125218-bbgu27nw/logs/debug.log b/wandb/run-20241115_125218-bbgu27nw/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..c67089cb150bfaace62e9c73cbab7f1e39fe350c --- /dev/null +++ b/wandb/run-20241115_125218-bbgu27nw/logs/debug.log @@ -0,0 +1,29 @@ +2024-11-15 12:52:18,155 INFO MainThread:2609854 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-11-15 12:52:18,155 INFO MainThread:2609854 [wandb_setup.py:_flush():79] Configure stats pid to 2609854 +2024-11-15 12:52:18,155 INFO MainThread:2609854 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-11-15 12:52:18,155 INFO MainThread:2609854 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-11-15 12:52:18,155 INFO MainThread:2609854 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-15 12:52:18,155 INFO MainThread:2609854 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-15 12:52:18,155 INFO MainThread:2609854 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_ftp.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_ftp.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_ftp.py'} +2024-11-15 12:52:18,155 INFO MainThread:2609854 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-15 12:52:18,156 INFO MainThread:2609854 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241115_125218-bbgu27nw/logs/debug.log +2024-11-15 12:52:18,156 INFO MainThread:2609854 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241115_125218-bbgu27nw/logs/debug-internal.log +2024-11-15 12:52:18,156 INFO MainThread:2609854 [wandb_init.py:init():621] calling init triggers +2024-11-15 12:52:18,156 INFO MainThread:2609854 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-11-15 12:52:18,156 INFO MainThread:2609854 [wandb_init.py:init():671] starting backend +2024-11-15 12:52:18,156 INFO MainThread:2609854 [wandb_init.py:init():675] sending inform_init request +2024-11-15 12:52:18,157 INFO MainThread:2609854 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-15 12:52:18,158 INFO MainThread:2609854 [wandb_init.py:init():688] backend started and connected +2024-11-15 12:52:18,161 INFO MainThread:2609854 [wandb_init.py:init():783] updated telemetry +2024-11-15 12:52:18,190 INFO MainThread:2609854 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-11-15 12:52:18,496 INFO MainThread:2609854 [wandb_init.py:init():867] starting run threads in backend +2024-11-15 12:52:18,618 INFO MainThread:2609854 [wandb_run.py:_console_start():2463] atexit reg +2024-11-15 12:52:18,618 INFO MainThread:2609854 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-11-15 12:52:18,618 INFO MainThread:2609854 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-11-15 12:52:18,618 INFO MainThread:2609854 [wandb_run.py:_redirect():2401] Redirects installed. +2024-11-15 12:52:18,620 INFO MainThread:2609854 [wandb_init.py:init():911] run started, returning control to user process +2024-11-15 12:52:18,620 INFO MainThread:2609854 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_deterministic84', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0, 'lr': 5e-06} +2024-11-15 12:57:20,376 INFO MainThread:2609854 [wandb_run.py:_config_callback():1390] config_cb None None {'vocab_size': 128256, 'max_position_embeddings': 131072, 'hidden_size': 3072, 'intermediate_size': 8192, 'num_hidden_layers': 28, 'num_attention_heads': 24, 'num_key_value_heads': 8, 'hidden_act': 'silu', 'initializer_range': 0.02, 'rms_norm_eps': 1e-05, 'pretraining_tp': 1, 'use_cache': True, 'rope_theta': 500000.0, 'rope_scaling': {'factor': 32.0, 'high_freq_factor': 4.0, 'low_freq_factor': 1.0, 'original_max_position_embeddings': 8192, 'rope_type': 'llama3'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 128, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'bfloat16', 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': True, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['LlamaForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 128000, 'pad_token_id': None, 'eos_token_id': 128001, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'meta-llama/Llama-3.2-3B', 'transformers_version': '4.45.1', 'model_type': 'llama', 'output_dir': './checkpoints/Llama-3.2-3B-FTP/babylm_shuffle_deterministic84_10M_seed0/runs', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': True, 'do_predict': False, 'eval_strategy': 'steps', 'prediction_loss_only': False, 'per_device_train_batch_size': 3, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 2, 'eval_accumulation_steps': None, 'eval_delay': 0, 'torch_empty_cache_steps': None, 'learning_rate': 5e-06, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 3, 'max_steps': -1, 'lr_scheduler_type': 'linear', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.1, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': './logs', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 1, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 100, 'save_total_limit': None, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 0, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': False, 'fp16': True, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 10, 'dataloader_num_workers': 0, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': './checkpoints/Llama-3.2-3B-FTP/babylm_shuffle_deterministic84_10M_seed0/runs', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': 'deepspeed_config/train_dp_config.json', 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'evaluation_strategy': 'steps', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False, 'use_liger_kernel': False, 'eval_use_gather_object': False} +2024-11-15 12:57:20,378 INFO MainThread:2609854 [wandb_config.py:__setitem__():154] config set model/num_parameters = 3212749824 - > +2024-11-15 12:57:20,378 INFO MainThread:2609854 [wandb_run.py:_config_callback():1390] config_cb model/num_parameters 3212749824 None diff --git a/wandb/run-20241118_085831-izbr4o0x/files/config.yaml b/wandb/run-20241118_085831-izbr4o0x/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7c6db7f1d23f8df1cdf8299fcab2bac6152ba311 --- /dev/null +++ b/wandb/run-20241118_085831-izbr4o0x/files/config.yaml @@ -0,0 +1,50 @@ +_wandb: + value: + cli_version: 0.18.5 + m: [] + python_version: 3.9.19 + t: + "1": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "2": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "3": + - 2 + - 13 + - 23 + - 55 + "4": 3.9.19 + "5": 0.18.5 + "6": 4.45.1 + "8": + - 5 + "12": 0.18.5 + "13": linux-x86_64 +batch_size: + value: 3 +epoch: + value: 3 +lr: + value: 5e-06 +perturbation: + value: shuffle_nondeterministic +seed: + value: 0 +train_set: + value: 10M diff --git a/wandb/run-20241118_085831-izbr4o0x/files/output.log b/wandb/run-20241118_085831-izbr4o0x/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..700b8c1b48f85d3f4534f7e7c069ec896a5a9bf6 --- /dev/null +++ b/wandb/run-20241118_085831-izbr4o0x/files/output.log @@ -0,0 +1,16 @@ +Downloading shards: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [02:32<00:00, 76.24s/it] +Loading checkpoint shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:04<00:00, 2.16s/it] +Map: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████| 16425/16425 [00:45<00:00, 358.72 examples/s] +Map: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████| 17013/17013 [00:46<00:00, 368.15 examples/s] +tokenized_valid: Dataset({ + features: ['input_ids', 'attention_mask'], + num_rows: 1000 +}) +/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +[2024-11-18 09:03:05,251] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-11-18 09:03:12,456] [INFO] [comm.py:652:init_distributed] cdb=None +Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination +Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root... +Loading extension module cpu_adam... +Time to load cpu_adam op: 7.149499177932739 seconds diff --git a/wandb/run-20241118_085831-izbr4o0x/files/wandb-metadata.json b/wandb/run-20241118_085831-izbr4o0x/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c550f6817b89aa9f40e7cabd848f0687240f853f --- /dev/null +++ b/wandb/run-20241118_085831-izbr4o0x/files/wandb-metadata.json @@ -0,0 +1,97 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-11-18T13:58:31.938999Z", + "args": [ + "--perturbation", + "shuffle_nondeterministic", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "3", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_ftp.py", + "codePath": "train/train_ftp.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_ftp.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA RTX A6000", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888559353856", + "used": "1786847916032" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + } + ], + "cudaVersion": "11.8" +} \ No newline at end of file diff --git a/wandb/run-20241118_085831-izbr4o0x/files/wandb-summary.json b/wandb/run-20241118_085831-izbr4o0x/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..e8ae1757e62e760d5988c92a580e7d045310389f --- /dev/null +++ b/wandb/run-20241118_085831-izbr4o0x/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":43009}} \ No newline at end of file diff --git a/wandb/run-20241118_085831-izbr4o0x/logs/debug-internal.log b/wandb/run-20241118_085831-izbr4o0x/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..e013d0aa9ef60d18a6dd9c25b5974728624fc5d9 --- /dev/null +++ b/wandb/run-20241118_085831-izbr4o0x/logs/debug-internal.log @@ -0,0 +1,18 @@ +{"time":"2024-11-18T08:58:31.941411288-05:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-11-18T08:58:31.941421148-05:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241118_085831-izbr4o0x/logs/debug-core.log"} +{"time":"2024-11-18T08:58:32.052765225-05:00","level":"INFO","msg":"created new stream","id":"izbr4o0x"} +{"time":"2024-11-18T08:58:32.052791555-05:00","level":"INFO","msg":"stream: started","id":"izbr4o0x"} +{"time":"2024-11-18T08:58:32.052882046-05:00","level":"INFO","msg":"sender: started","stream_id":"izbr4o0x"} +{"time":"2024-11-18T08:58:32.052851655-05:00","level":"INFO","msg":"handler: started","stream_id":{"value":"izbr4o0x"}} +{"time":"2024-11-18T08:58:32.052831215-05:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"izbr4o0x"}} +{"time":"2024-11-18T08:58:32.435798879-05:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-11-18T13:59:32.835724526-05:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/yaning1001-dartmouth-college/exp-impo-shuffle/izbr4o0x/file_stream"} +{"time":"2024-11-18T20:55:21.832201771-05:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2024-11-18T20:55:22.036166856-05:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2024-11-18T20:55:22.65077132-05:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2024-11-18T20:55:22.67266896-05:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"uploading summary","runtime_seconds":0.021663119}],"total_operations":1}} +{"time":"2024-11-18T20:55:24.000886546-05:00","level":"INFO","msg":"stream: closing","id":"izbr4o0x"} +{"time":"2024-11-18T20:55:24.000931207-05:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"izbr4o0x"}} +{"time":"2024-11-18T20:55:24.000961817-05:00","level":"INFO","msg":"sender: closed","stream_id":"izbr4o0x"} +{"time":"2024-11-18T20:55:24.000956237-05:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"izbr4o0x"}} +{"time":"2024-11-18T20:55:24.001114028-05:00","level":"INFO","msg":"stream: closed","id":"izbr4o0x"} diff --git a/wandb/run-20241118_085831-izbr4o0x/logs/debug.log b/wandb/run-20241118_085831-izbr4o0x/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..6a19e89d74eaaef896fb8055d67e1c9e37cdbce8 --- /dev/null +++ b/wandb/run-20241118_085831-izbr4o0x/logs/debug.log @@ -0,0 +1,33 @@ +2024-11-18 08:58:31,935 INFO MainThread:2672321 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-11-18 08:58:31,935 INFO MainThread:2672321 [wandb_setup.py:_flush():79] Configure stats pid to 2672321 +2024-11-18 08:58:31,935 INFO MainThread:2672321 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-11-18 08:58:31,936 INFO MainThread:2672321 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-11-18 08:58:31,936 INFO MainThread:2672321 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-18 08:58:31,936 INFO MainThread:2672321 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-18 08:58:31,936 INFO MainThread:2672321 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_ftp.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_ftp.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_ftp.py'} +2024-11-18 08:58:31,936 INFO MainThread:2672321 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-18 08:58:31,936 INFO MainThread:2672321 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241118_085831-izbr4o0x/logs/debug.log +2024-11-18 08:58:31,936 INFO MainThread:2672321 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241118_085831-izbr4o0x/logs/debug-internal.log +2024-11-18 08:58:31,936 INFO MainThread:2672321 [wandb_init.py:init():621] calling init triggers +2024-11-18 08:58:31,936 INFO MainThread:2672321 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-11-18 08:58:31,936 INFO MainThread:2672321 [wandb_init.py:init():671] starting backend +2024-11-18 08:58:31,936 INFO MainThread:2672321 [wandb_init.py:init():675] sending inform_init request +2024-11-18 08:58:31,938 INFO MainThread:2672321 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-18 08:58:31,938 INFO MainThread:2672321 [wandb_init.py:init():688] backend started and connected +2024-11-18 08:58:31,941 INFO MainThread:2672321 [wandb_init.py:init():783] updated telemetry +2024-11-18 08:58:31,979 INFO MainThread:2672321 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-11-18 08:58:32,433 INFO MainThread:2672321 [wandb_init.py:init():867] starting run threads in backend +2024-11-18 08:58:32,520 INFO MainThread:2672321 [wandb_run.py:_console_start():2463] atexit reg +2024-11-18 08:58:32,520 INFO MainThread:2672321 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-11-18 08:58:32,520 INFO MainThread:2672321 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-11-18 08:58:32,520 INFO MainThread:2672321 [wandb_run.py:_redirect():2401] Redirects installed. +2024-11-18 08:58:32,522 INFO MainThread:2672321 [wandb_init.py:init():911] run started, returning control to user process +2024-11-18 08:58:32,522 INFO MainThread:2672321 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'shuffle_nondeterministic', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0, 'lr': 5e-06} +2024-11-18 20:55:21,528 INFO MainThread:2672321 [wandb_run.py:_finish():2158] finishing run yaning1001-dartmouth-college/exp-impo-shuffle/izbr4o0x +2024-11-18 20:55:21,670 INFO MainThread:2672321 [wandb_run.py:_atexit_cleanup():2426] got exitcode: 0 +2024-11-18 20:55:21,671 INFO MainThread:2672321 [wandb_run.py:_restore():2408] restore +2024-11-18 20:55:21,671 INFO MainThread:2672321 [wandb_run.py:_restore():2414] restore done +2024-11-18 20:55:23,820 INFO MainThread:2672321 [wandb_run.py:_footer_history_summary_info():3975] rendering history +2024-11-18 20:55:23,821 INFO MainThread:2672321 [wandb_run.py:_footer_history_summary_info():4007] rendering summary +2024-11-18 20:55:23,961 INFO MainThread:2672321 [wandb_run.py:_footer_sync_info():3934] logging synced files diff --git a/wandb/run-20241129_235241-hx37w012/files/requirements.txt b/wandb/run-20241129_235241-hx37w012/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..95a931302e269cc9e4fa5b719b6511f176ee2416 --- /dev/null +++ b/wandb/run-20241129_235241-hx37w012/files/requirements.txt @@ -0,0 +1,147 @@ +funcsigs==1.0.2 +sentry-sdk==2.17.0 +multiprocess==0.70.16 +numpy==1.26.2 +pluralizer==1.2.0 +debugpy==1.6.7 +nvidia-cudnn-cu11==8.5.0.96 +deepspeed==0.15.2 +data==0.4 +pandas==2.1.3 +tomli==2.0.1 +charset-normalizer==3.3.2 +attrs==24.2.0 +aiosignal==1.3.1 +fsspec==2023.10.0 +nvidia-cusparse-cu11==11.7.4.91 +zipp==3.12.0 +mypy-extensions==1.0.0 +datasets==3.0.1 +joblib==1.3.2 +hjson==3.1.0 +traitlets==5.7.1 +stack-data==0.6.0 +transformers==4.45.1 +sympy==1.11.1 +Pygments==2.15.0 +docker-pycreds==0.4.0 +dill==0.3.8 +wheel==0.44.0 +prompt-toolkit==3.0.30 +parso==0.8.3 +ipykernel==6.23.1 +pyarrow==17.0.0 +certifi==2023.11.17 +nvidia-cufft-cu11==10.9.0.58 +six==1.16.0 +pydantic==2.9.2 +click==8.1.7 +nest-asyncio==1.5.6 +gmpy2==2.1.0 +matplotlib==3.8.2 +scipy==1.11.4 +typing_extensions==4.12.2 +statsmodels==0.14.0 +huggingface-hub==0.25.0 +frozenlist==1.4.1 +gpustat==1.1.1 +nvidia-nvtx-cu11==11.7.91 +safetensors==0.4.5 +stanza==1.9.2 +decorator==5.1.1 +seaborn==0.13.0 +sentencepiece==0.2.0 +PyYAML==6.0.1 +black==24.8.0 +protobuf==4.25.1 +pickleshare==0.7.5 +peft==0.13.0 +triton==2.0.0 +nvidia-cuda-runtime-cu11==11.7.99 +Jinja2==3.1.2 +nvidia-cusolver-cu11==11.4.0.1 +executing==1.2.0 +jupyter_client==8.1.0 +pluggy==1.3.0 +cmake==3.30.3 +pytz==2023.3.post1 +aiohappyeyeballs==2.4.2 +kiwisolver==1.4.5 +py-cpuinfo==9.0.0 +Pillow==10.1.0 +ptyprocess==0.7.0 +importlib_resources==6.4.5 +GitPython==3.1.43 +importlib-metadata==6.0.0 +iniconfig==2.0.0 +scikit-learn==1.3.2 +exceptiongroup==1.1.0 +networkx==2.8.6 +accelerate==1.0.0 +nltk==3.8.1 +shutilwhich==1.1.0 +fonttools==4.45.1 +future==0.18.3 +aiohttp==3.10.6 +wcwidth==0.2.5 +idna==3.6 +filelock==3.12.2 +pathspec==0.12.1 +jupyter_core==5.1.0 +lit==18.1.8 +nvidia-curand-cu11==10.2.10.91 +nvidia-cublas-cu11==11.10.3.66 +nvidia-ml-py==12.560.30 +msgpack==1.1.0 +python-dateutil==2.8.2 +blessed==1.20.0 +packaging==23.0 +gitdb==4.0.11 +yarl==1.13.0 +emoji==2.8.0 +tzdata==2023.3 +cycler==0.12.1 +tornado==6.2 +backcall==0.2.0 +plotnine==0.12.4 +ninja==1.11.1.1 +latex==0.7.0 +wandb==0.18.5 +setproctitle==1.3.3 +threadpoolctl==3.2.0 +requests==2.32.3 +pyparsing==3.1.1 +smmap==5.0.1 +pyzmq==23.0.0 +async-timeout==4.0.3 +annotated-types==0.7.0 +matplotlib-inline==0.1.6 +latexcodec==1.0.0 +ipython==8.0.0 +patsy==0.5.3 +contourpy==1.2.0 +multidict==6.1.0 +mizani==0.9.3 +urllib3==2.1.0 +tokenizers==0.20.0 +MarkupSafe==2.1.2 +pip==24.2 +pexpect==4.8.0 +tqdm==4.66.5 +jedi==0.18.2 +pydantic_core==2.23.4 +tempdir==0.7.1 +mpmath==1.2.1 +setuptools==72.1.0 +pytest==7.4.3 +pure-eval==0.2.2 +psutil==5.9.1 +comm==0.1.2 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +regex==2023.10.3 +platformdirs==2.5.2 +asttokens==2.2.1 +torch==2.0.0 +nvidia-nccl-cu11==2.14.3 +xxhash==3.5.0 diff --git a/wandb/run-20241129_235241-hx37w012/files/wandb-metadata.json b/wandb/run-20241129_235241-hx37w012/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..f09e387a2b2f43daf676840f557f456d214a8ca2 --- /dev/null +++ b/wandb/run-20241129_235241-hx37w012/files/wandb-metadata.json @@ -0,0 +1,97 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-11-30T04:52:41.146657Z", + "args": [ + "--perturbation", + "reverse_control", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "3", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_gpt2.py", + "codePath": "train/train_gpt2.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_gpt2.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA RTX A6000", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888559353856", + "used": "1719074058240" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + } + ], + "cudaVersion": "11.8" +} \ No newline at end of file diff --git a/wandb/run-20241129_235241-hx37w012/logs/debug-core.log b/wandb/run-20241129_235241-hx37w012/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..29fd1aa4868c88aa6e8ad37f667ff6400993bc93 --- /dev/null +++ b/wandb/run-20241129_235241-hx37w012/logs/debug-core.log @@ -0,0 +1,24 @@ +{"time":"2024-11-29T23:52:40.794147496-05:00","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmpkrexe9wt/port-3200481.txt","pid":3200481,"debug":false,"disable-analytics":false} +{"time":"2024-11-29T23:52:40.794182726-05:00","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false} +{"time":"2024-11-29T23:52:40.795140224-05:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":3200481} +{"time":"2024-11-29T23:52:40.795155284-05:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":43909,"Zone":""}} +{"time":"2024-11-29T23:52:40.801433856-05:00","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmpcp99_atc/port-3200479.txt","pid":3200479,"debug":false,"disable-analytics":false} +{"time":"2024-11-29T23:52:40.801457826-05:00","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false} +{"time":"2024-11-29T23:52:40.80197699-05:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":3200479} +{"time":"2024-11-29T23:52:40.80198148-05:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":35697,"Zone":""}} +{"time":"2024-11-29T23:52:40.808908666-05:00","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmpeo16ghb_/port-3200480.txt","pid":3200480,"debug":false,"disable-analytics":false} +{"time":"2024-11-29T23:52:40.808956096-05:00","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false} +{"time":"2024-11-29T23:52:40.809514601-05:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":3200480} +{"time":"2024-11-29T23:52:40.809524331-05:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":40771,"Zone":""}} +{"time":"2024-11-29T23:52:40.985268342-05:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:45208"} +{"time":"2024-11-29T23:52:40.993322078-05:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:46582"} +{"time":"2024-11-29T23:52:41.002111259-05:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:37004"} +{"time":"2024-11-29T23:52:41.149236286-05:00","level":"INFO","msg":"handleInformInit: received","streamId":"hx37w012","id":"127.0.0.1:46582"} +{"time":"2024-11-29T23:52:41.149310347-05:00","level":"INFO","msg":"handleInformInit: received","streamId":"3vsgtb8n","id":"127.0.0.1:45208"} +{"time":"2024-11-29T23:52:41.158457462-05:00","level":"INFO","msg":"handleInformInit: received","streamId":"3ftwr60m","id":"127.0.0.1:37004"} +{"time":"2024-11-29T23:52:41.257153555-05:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"hx37w012","id":"127.0.0.1:46582"} +{"time":"2024-11-29T23:52:41.257268116-05:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"3vsgtb8n","id":"127.0.0.1:45208"} +{"time":"2024-11-29T23:52:41.26517705-05:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"3ftwr60m","id":"127.0.0.1:37004"} +{"time":"2024-11-30T00:02:00.014655319-05:00","level":"INFO","msg":"Parent process exited, terminating service process."} +{"time":"2024-11-30T00:02:00.213898401-05:00","level":"INFO","msg":"Parent process exited, terminating service process."} +{"time":"2024-11-30T00:02:00.316781185-05:00","level":"INFO","msg":"Parent process exited, terminating service process."} diff --git a/wandb/run-20241129_235241-hx37w012/logs/debug-internal.log b/wandb/run-20241129_235241-hx37w012/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..048b4146c522196bdaf98fd4cd1e85f159925152 --- /dev/null +++ b/wandb/run-20241129_235241-hx37w012/logs/debug-internal.log @@ -0,0 +1,10 @@ +{"time":"2024-11-29T23:52:41.149412228-05:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-11-29T23:52:41.149427438-05:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241129_235241-hx37w012/logs/debug-core.log"} +{"time":"2024-11-29T23:52:41.257094334-05:00","level":"INFO","msg":"created new stream","id":"hx37w012"} +{"time":"2024-11-29T23:52:41.257144065-05:00","level":"INFO","msg":"stream: started","id":"hx37w012"} +{"time":"2024-11-29T23:52:41.257191785-05:00","level":"INFO","msg":"sender: started","stream_id":"hx37w012"} +{"time":"2024-11-29T23:52:41.257165705-05:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"hx37w012"}} +{"time":"2024-11-29T23:52:41.257188865-05:00","level":"INFO","msg":"handler: started","stream_id":{"value":"hx37w012"}} +{"time":"2024-11-29T23:52:41.433563231-05:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-11-30T00:01:56.818216555-05:00","level":"ERROR","msg":"HTTP error","status":404,"method":"POST","url":"https://api.wandb.ai/files/yaning1001-dartmouth-college/exp-impo-shuffle/hx37w012/file_stream"} +{"time":"2024-11-30T00:01:56.818281985-05:00","level":"ERROR+4","msg":"filestream: fatal error: filestream: failed to upload: 404 Not Found path=files/yaning1001-dartmouth-college/exp-impo-shuffle/hx37w012/file_stream: {\"error\":\"run exp-impo-shuffle/hx37w012 not found while streaming file\"}"} diff --git a/wandb/run-20241129_235241-hx37w012/logs/debug.log b/wandb/run-20241129_235241-hx37w012/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..2e44e92ae07ed0ffa0e8dd10d1b8e7818ade8eb6 --- /dev/null +++ b/wandb/run-20241129_235241-hx37w012/logs/debug.log @@ -0,0 +1,29 @@ +2024-11-29 23:52:41,144 INFO MainThread:3200479 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-11-29 23:52:41,145 INFO MainThread:3200479 [wandb_setup.py:_flush():79] Configure stats pid to 3200479 +2024-11-29 23:52:41,145 INFO MainThread:3200479 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-11-29 23:52:41,145 INFO MainThread:3200479 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-11-29 23:52:41,145 INFO MainThread:3200479 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-29 23:52:41,145 INFO MainThread:3200479 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-29 23:52:41,145 INFO MainThread:3200479 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_gpt2.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_gpt2.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_gpt2.py'} +2024-11-29 23:52:41,145 INFO MainThread:3200479 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-29 23:52:41,145 INFO MainThread:3200479 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241129_235241-hx37w012/logs/debug.log +2024-11-29 23:52:41,145 INFO MainThread:3200479 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241129_235241-hx37w012/logs/debug-internal.log +2024-11-29 23:52:41,145 INFO MainThread:3200479 [wandb_init.py:init():621] calling init triggers +2024-11-29 23:52:41,145 INFO MainThread:3200479 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-11-29 23:52:41,145 INFO MainThread:3200479 [wandb_init.py:init():671] starting backend +2024-11-29 23:52:41,145 INFO MainThread:3200479 [wandb_init.py:init():675] sending inform_init request +2024-11-29 23:52:41,146 INFO MainThread:3200479 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-29 23:52:41,146 INFO MainThread:3200479 [wandb_init.py:init():688] backend started and connected +2024-11-29 23:52:41,149 INFO MainThread:3200479 [wandb_init.py:init():783] updated telemetry +2024-11-29 23:52:41,178 INFO MainThread:3200479 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-11-29 23:52:41,430 INFO MainThread:3200479 [wandb_init.py:init():867] starting run threads in backend +2024-11-29 23:52:41,545 INFO MainThread:3200479 [wandb_run.py:_console_start():2463] atexit reg +2024-11-29 23:52:41,545 INFO MainThread:3200479 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-11-29 23:52:41,545 INFO MainThread:3200479 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-11-29 23:52:41,545 INFO MainThread:3200479 [wandb_run.py:_redirect():2401] Redirects installed. +2024-11-29 23:52:41,547 INFO MainThread:3200479 [wandb_init.py:init():911] run started, returning control to user process +2024-11-29 23:52:41,547 INFO MainThread:3200479 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_control', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0, 'lr': 5e-06} +2024-11-29 23:54:54,073 INFO MainThread:3200479 [wandb_run.py:_config_callback():1390] config_cb None None {'vocab_size': 50257, 'n_positions': 1024, 'n_embd': 768, 'n_layer': 12, 'n_head': 12, 'n_inner': None, 'activation_function': 'gelu_new', 'resid_pdrop': 0.1, 'embd_pdrop': 0.1, 'attn_pdrop': 0.1, 'layer_norm_epsilon': 1e-05, 'initializer_range': 0.02, 'summary_type': 'cls_index', 'summary_use_proj': True, 'summary_activation': None, 'summary_first_dropout': 0.1, 'summary_proj_to_labels': True, 'scale_attn_weights': True, 'use_cache': True, 'scale_attn_by_inverse_layer_idx': False, 'reorder_and_upcast_attn': False, 'bos_token_id': 50256, 'eos_token_id': 50256, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': None, 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': True, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['GPT2LMHeadModel'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'pad_token_id': None, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': {'text-generation': {'do_sample': True, 'max_length': 50}}, 'problem_type': None, '_name_or_path': 'gpt2', 'transformers_version': '4.45.1', 'model_type': 'gpt2', 'n_ctx': 1024, 'output_dir': './checkpoints/GPT2/babylm_reverse_control_10M_seed0/runs', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': True, 'do_predict': False, 'eval_strategy': 'steps', 'prediction_loss_only': False, 'per_device_train_batch_size': 3, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 2, 'eval_accumulation_steps': None, 'eval_delay': 0, 'torch_empty_cache_steps': None, 'learning_rate': 5e-06, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 3, 'max_steps': -1, 'lr_scheduler_type': 'linear', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.1, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': './logs', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 1, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 100, 'save_total_limit': None, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 0, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': False, 'fp16': True, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 10, 'dataloader_num_workers': 0, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': './checkpoints/GPT2/babylm_reverse_control_10M_seed0/runs', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': 'deepspeed_config/train_dp_config.json', 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'evaluation_strategy': 'steps', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False, 'use_liger_kernel': False, 'eval_use_gather_object': False} +2024-11-29 23:54:54,075 INFO MainThread:3200479 [wandb_config.py:__setitem__():154] config set model/num_parameters = 124439808 - > +2024-11-29 23:54:54,075 INFO MainThread:3200479 [wandb_run.py:_config_callback():1390] config_cb model/num_parameters 124439808 None diff --git a/wandb/run-20241130_011200-7p4fy9o8/files/config.yaml b/wandb/run-20241130_011200-7p4fy9o8/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..164653596defa3a070943f5b9f7cd18e7f2d95cc --- /dev/null +++ b/wandb/run-20241130_011200-7p4fy9o8/files/config.yaml @@ -0,0 +1,50 @@ +_wandb: + value: + cli_version: 0.18.5 + m: [] + python_version: 3.9.19 + t: + "1": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "2": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + - 55 + - 71 + - 98 + "3": + - 2 + - 13 + - 23 + - 55 + "4": 3.9.19 + "5": 0.18.5 + "6": 4.45.1 + "8": + - 5 + "12": 0.18.5 + "13": linux-x86_64 +batch_size: + value: 3 +epoch: + value: 3 +lr: + value: 5e-06 +perturbation: + value: reverse_partial +seed: + value: 0 +train_set: + value: 10M diff --git a/wandb/run-20241130_011200-7p4fy9o8/files/output.log b/wandb/run-20241130_011200-7p4fy9o8/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..6afbd0e9d831e535f87afc94f5f8142cd472add9 --- /dev/null +++ b/wandb/run-20241130_011200-7p4fy9o8/files/output.log @@ -0,0 +1,17 @@ +Map: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 17519/17519 [00:32<00:00, 534.06 examples/s] +Map: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 18140/18140 [00:35<00:00, 508.18 examples/s] +tokenized_valid: Dataset({ + features: ['input_ids', 'attention_mask'], + num_rows: 1000 +}) +/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/lib/python3.9/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead + warnings.warn( +[2024-11-30 01:13:37,183] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2024-11-30 01:13:40,303] [INFO] [comm.py:652:init_distributed] cdb=None +Installed CUDA version 11.8 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination +Using /home/chunhui/.cache/torch_extensions/py39_cu117 as PyTorch extensions root... +Emitting ninja build file /home/chunhui/.cache/torch_extensions/py39_cu117/cpu_adam/build.ninja... +Building extension module cpu_adam... +Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +Loading extension module cpu_adam... +Time to load cpu_adam op: 2.6265251636505127 seconds diff --git a/wandb/run-20241130_011200-7p4fy9o8/files/wandb-metadata.json b/wandb/run-20241130_011200-7p4fy9o8/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..9600df9048ca8dabd15cda40b2729c804f6b1e41 --- /dev/null +++ b/wandb/run-20241130_011200-7p4fy9o8/files/wandb-metadata.json @@ -0,0 +1,97 @@ +{ + "os": "Linux-5.4.0-162-generic-x86_64-with-glibc2.31", + "python": "3.9.19", + "startedAt": "2024-11-30T06:12:00.582313Z", + "args": [ + "--perturbation", + "reverse_partial", + "--train_set", + "10M", + "--batch_size", + "3", + "--epoch", + "3", + "--seed", + "0" + ], + "program": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_gpt2.py", + "codePath": "train/train_gpt2.py", + "git": { + "remote": "git@hf.co:Yaning1001/Impossible_llm.git", + "commit": "ed716cdcfcdea02b67f7ed0f3504c2b1c8b737c4" + }, + "email": "yaning1001@gmail.com", + "root": "/mnt/ssd3/chunhui/yaning/project/impossible_llm/train", + "host": "mms-large-2", + "username": "chunhui", + "executable": "/mnt/ssd3/chunhui/miniconda/envs/impossible_llm/bin/python", + "codePathLocal": "train_gpt2.py", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA RTX A6000", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888559353856", + "used": "1720385073152" + } + }, + "memory": { + "total": "202617098240" + }, + "cpu": { + "count": 32, + "countLogical": 64 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + }, + { + "name": "NVIDIA RTX A6000", + "memoryTotal": "51527024640", + "cudaCores": 10752, + "architecture": "Ampere" + } + ], + "cudaVersion": "11.8" +} \ No newline at end of file diff --git a/wandb/run-20241130_011200-7p4fy9o8/files/wandb-summary.json b/wandb/run-20241130_011200-7p4fy9o8/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..26d44b8ab182e0f622a823169a56c1ed3ca76166 --- /dev/null +++ b/wandb/run-20241130_011200-7p4fy9o8/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":3550}} \ No newline at end of file diff --git a/wandb/run-20241130_011200-7p4fy9o8/logs/debug-internal.log b/wandb/run-20241130_011200-7p4fy9o8/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..a8975a358ba7f9f391e3bf6f2e9ea43d830b0abb --- /dev/null +++ b/wandb/run-20241130_011200-7p4fy9o8/logs/debug-internal.log @@ -0,0 +1,17 @@ +{"time":"2024-11-30T01:12:00.584497778-05:00","level":"INFO","msg":"using version","core version":"0.18.5"} +{"time":"2024-11-30T01:12:00.584512378-05:00","level":"INFO","msg":"created symlink","path":"/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241130_011200-7p4fy9o8/logs/debug-core.log"} +{"time":"2024-11-30T01:12:00.690595233-05:00","level":"INFO","msg":"created new stream","id":"7p4fy9o8"} +{"time":"2024-11-30T01:12:00.690619613-05:00","level":"INFO","msg":"stream: started","id":"7p4fy9o8"} +{"time":"2024-11-30T01:12:00.690681993-05:00","level":"INFO","msg":"sender: started","stream_id":"7p4fy9o8"} +{"time":"2024-11-30T01:12:00.690644643-05:00","level":"INFO","msg":"handler: started","stream_id":{"value":"7p4fy9o8"}} +{"time":"2024-11-30T01:12:00.690641003-05:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"7p4fy9o8"}} +{"time":"2024-11-30T01:12:00.859481271-05:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-11-30T02:11:11.340074116-05:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2024-11-30T02:11:11.340981582-05:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2024-11-30T02:11:11.766871033-05:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2024-11-30T02:11:11.948700811-05:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2024-11-30T02:11:12.958514865-05:00","level":"INFO","msg":"stream: closing","id":"7p4fy9o8"} +{"time":"2024-11-30T02:11:12.958555006-05:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"7p4fy9o8"}} +{"time":"2024-11-30T02:11:12.958588266-05:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"7p4fy9o8"}} +{"time":"2024-11-30T02:11:12.958624136-05:00","level":"INFO","msg":"sender: closed","stream_id":"7p4fy9o8"} +{"time":"2024-11-30T02:11:12.958703497-05:00","level":"INFO","msg":"stream: closed","id":"7p4fy9o8"} diff --git a/wandb/run-20241130_011200-7p4fy9o8/logs/debug.log b/wandb/run-20241130_011200-7p4fy9o8/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..c0dd14337436a41a1a70f244c2ecc74c03474f54 --- /dev/null +++ b/wandb/run-20241130_011200-7p4fy9o8/logs/debug.log @@ -0,0 +1,33 @@ +2024-11-30 01:12:00,579 INFO MainThread:3204336 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5 +2024-11-30 01:12:00,580 INFO MainThread:3204336 [wandb_setup.py:_flush():79] Configure stats pid to 3204336 +2024-11-30 01:12:00,580 INFO MainThread:3204336 [wandb_setup.py:_flush():79] Loading settings from /home/chunhui/.config/wandb/settings +2024-11-30 01:12:00,580 INFO MainThread:3204336 [wandb_setup.py:_flush():79] Loading settings from /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/settings +2024-11-30 01:12:00,580 INFO MainThread:3204336 [wandb_setup.py:_flush():79] Loading settings from environment variables: {} +2024-11-30 01:12:00,580 INFO MainThread:3204336 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None} +2024-11-30 01:12:00,580 INFO MainThread:3204336 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train/train_gpt2.py', 'program_abspath': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_gpt2.py', 'program': '/mnt/ssd3/chunhui/yaning/project/impossible_llm/train/train_gpt2.py'} +2024-11-30 01:12:00,580 INFO MainThread:3204336 [wandb_setup.py:_flush():79] Applying login settings: {} +2024-11-30 01:12:00,580 INFO MainThread:3204336 [wandb_init.py:_log_setup():534] Logging user logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241130_011200-7p4fy9o8/logs/debug.log +2024-11-30 01:12:00,580 INFO MainThread:3204336 [wandb_init.py:_log_setup():535] Logging internal logs to /mnt/ssd3/chunhui/yaning/project/impossible_llm/train/wandb/run-20241130_011200-7p4fy9o8/logs/debug-internal.log +2024-11-30 01:12:00,580 INFO MainThread:3204336 [wandb_init.py:init():621] calling init triggers +2024-11-30 01:12:00,580 INFO MainThread:3204336 [wandb_init.py:init():628] wandb.init called with sweep_config: {} +config: {} +2024-11-30 01:12:00,580 INFO MainThread:3204336 [wandb_init.py:init():671] starting backend +2024-11-30 01:12:00,580 INFO MainThread:3204336 [wandb_init.py:init():675] sending inform_init request +2024-11-30 01:12:00,581 INFO MainThread:3204336 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-30 01:12:00,582 INFO MainThread:3204336 [wandb_init.py:init():688] backend started and connected +2024-11-30 01:12:00,585 INFO MainThread:3204336 [wandb_init.py:init():783] updated telemetry +2024-11-30 01:12:00,613 INFO MainThread:3204336 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout +2024-11-30 01:12:00,856 INFO MainThread:3204336 [wandb_init.py:init():867] starting run threads in backend +2024-11-30 01:12:00,949 INFO MainThread:3204336 [wandb_run.py:_console_start():2463] atexit reg +2024-11-30 01:12:00,949 INFO MainThread:3204336 [wandb_run.py:_redirect():2311] redirect: wrap_raw +2024-11-30 01:12:00,949 INFO MainThread:3204336 [wandb_run.py:_redirect():2376] Wrapping output streams. +2024-11-30 01:12:00,949 INFO MainThread:3204336 [wandb_run.py:_redirect():2401] Redirects installed. +2024-11-30 01:12:00,951 INFO MainThread:3204336 [wandb_init.py:init():911] run started, returning control to user process +2024-11-30 01:12:00,951 INFO MainThread:3204336 [wandb_run.py:_config_callback():1390] config_cb None None {'perturbation': 'reverse_partial', 'train_set': '10M', 'batch_size': 3, 'epoch': 3, 'seed': 0, 'lr': 5e-06} +2024-11-30 02:11:11,338 INFO MainThread:3204336 [wandb_run.py:_finish():2158] finishing run yaning1001-dartmouth-college/exp-impo-reverse/7p4fy9o8 +2024-11-30 02:11:11,339 INFO MainThread:3204336 [wandb_run.py:_atexit_cleanup():2426] got exitcode: 0 +2024-11-30 02:11:11,339 INFO MainThread:3204336 [wandb_run.py:_restore():2408] restore +2024-11-30 02:11:11,339 INFO MainThread:3204336 [wandb_run.py:_restore():2414] restore done +2024-11-30 02:11:12,952 INFO MainThread:3204336 [wandb_run.py:_footer_history_summary_info():3975] rendering history +2024-11-30 02:11:12,952 INFO MainThread:3204336 [wandb_run.py:_footer_history_summary_info():4007] rendering summary +2024-11-30 02:11:12,957 INFO MainThread:3204336 [wandb_run.py:_footer_sync_info():3934] logging synced files