Saving weights and logs of step 50000

checkpoint_50000

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3a84b4fd88da1f2759c96727ee823c17d3316e9bb0b83ac134d758d04f738723
+oid sha256:b4b3bf816e2cbc7650c0a002c813cc9fc134e3d1aed78679a7f08cc2e70bc304
 size 2886112612

events.out.tfevents.1636287687.t1v-n-ba7aa1a5-w-0.13180.0.v2

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c72c4d7407936638fc79b65a5644f6d29dfe58ef25932a44a8cd03dcacee2696
-size 7421069
+oid sha256:acdd1d6f4159c76941ac48d6cec8f0992e5c4de06d320d2b71a8a9ef5b2353ae
+size 8911911

events.out.tfevents.1636318087.t1v-n-ba7aa1a5-w-0.20231.0.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff6e8edb09c1a4da70a68ce6e234d81e713fac71cb512923e146133875bc8d13
+size 401

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:92b4eb4b41a4fdd29fa296ca969236da8c1c0c923dab34bfff0448484f2a8228
+oid sha256:9cac41482764ea6b546c23005d6f0fc5447d39960bb9edbe4a412eaef5f6ebee
 size 962037499

muril-bigbird/flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0b4ee04dcd8c60be62794b6e99438b6a9caf8cd063f11f69236b0652ad6e1dd9
+oid sha256:92b4eb4b41a4fdd29fa296ca969236da8c1c0c923dab34bfff0448484f2a8228
 size 962037499

run_muril_bb_flax.sh

@@ -13,16 +13,16 @@ python run_muril_bb_flax_mlm.py \
   --overwrite_output_dir \
   --do_train \
   --do_eval \
-  --eval_steps 15000 \
+  --eval_steps 5000 \
   --per_device_train_batch_size 1 \
   --per_device_eval_batch_size 2 \
-  --learning_rate 1e-4 \
+  --learning_rate 5e-5 \
   --weight_decay 0.01 \
   --num_train_epochs 3 \
-  --warmup_steps 10000 \
-  --logging_steps 1000 \
+  --warmup_steps 0 \
+  --logging_steps 500 \
   --save_steps 50000 \
   --save_total_limit 1 \
   --seed 18 \
   --push_to_hub \
-  --run_name "en-ta-hi-te"
+  --run_name "en-ta-hi-te-from-50k"

run_muril_bb_flax_mlm.py

@@ -343,10 +343,10 @@ if __name__ == "__main__":
         files = p.glob("*/*.parquet")
         files = [str(x) for x in files]
         random.shuffle(files)
-        files = files[:50]
+        files = files[50:100]
         datasets= load_dataset("parquet", data_files=files, split="train", cache_dir="/dev/shm")
         datasets = datasets.shuffle(seed=training_args.seed)
-        datasets = datasets.train_test_split(test_size = 0.015, seed=training_args.seed)
+        datasets = datasets.train_test_split(test_size = 0.02, seed=training_args.seed)
         datasets["validation"] = datasets["test"]
 
     elif data_args.dataset_name is not None:
@@ -612,7 +612,7 @@ if __name__ == "__main__":
         state = restore_checkpoint(training_args.resume_from_checkpoint, state)
         resume_step = mb_item(state.step.item())
     else:
-        resume_step = 0
+        resume_step = 50000
 
     # Define gradient update step fn
     def train_step(state, batch, dropout_rng):

untrained_flax_model.msgpack

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b4ee04dcd8c60be62794b6e99438b6a9caf8cd063f11f69236b0652ad6e1dd9
+size 962037499

wandb/debug-internal.log

@@ -1 +1 @@
-run-20211107_122127-18mmtpt3/logs/debug-internal.log
+run-20211107_204808-2ns3lo5n/logs/debug-internal.log

wandb/debug.log

@@ -1 +1 @@
-run-20211107_122127-18mmtpt3/logs/debug.log
+run-20211107_204808-2ns3lo5n/logs/debug.log

wandb/latest-run

@@ -1 +1 @@
-run-20211107_122127-18mmtpt3
+run-20211107_204808-2ns3lo5n

wandb/run-20211107_122127-18mmtpt3/files/config.yaml

@@ -23,6 +23,12 @@ _wandb:
       - 3
       - 11
       - 12
+      2:
+      - 1
+      - 2
+      - 3
+      - 11
+      - 12
       3:
       - 13
       4: 3.8.10

wandb/run-20211107_122127-18mmtpt3/files/output.log

@@ -11989,3 +11989,2450 @@ tcmalloc: large alloc 3161391104 bytes == 0x44822a000 @  0x7f2a10a2f680 0x7f2a10
 tcmalloc: large alloc 2886115328 bytes == 0x392038000 @  0x7f2a10a2f680 0x7f2a10a50824 0x5f9f21 0x7f286173ec6f 0x7f286173f1fe 0x5042c6 0x56bf09 0x56a0ba 0x5f6343 0x56cf2a 0x56a0ba 0x5f6343 0x56cf2a 0x5f6166 0x570e46 0x56a0ba 0x5f6343 0x56cf2a 0x56a0ba 0x68d5b7 0x67cd01 0x67cd7f 0x67ce21 0x67ef47 0x6b7242 0x6b75cd 0x7f2a108460b3 0x5fb18e
 [19:17:43] - INFO - absl - Saved checkpoint at ./checkpoint_50000
 Adding files tracked by Git LFS: ['checkpoint_50000']. This may take a bit of time if the files are large.
+[19:17:48] - WARNING - huggingface_hub.repository - Adding files tracked by Git LFS: ['checkpoint_50000']. This may take a bit of time if the files are large.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Training...:  84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌                                | 51015/60729 [7:03:33<48:19,  3.35it/s]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Training...:  86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████                             | 52000/60729 [7:11:21<1:10:34,  2.06it/s]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Training...:  87%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍                         | 53000/60729 [7:19:27<1:02:28,  2.06it/s]
+Step... (53000 | Loss: 1.6916450262069702, Learning Rate: 7.502715016016737e-05)
+Step... (45000 | Loss: 1.4846699237823486, Acc: 0.711020290851593):   0%|                                                                                                                                                    | 0/3 [7:20:51<?, ?it/s]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Training...:  89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌                      | 54000/60729 [7:27:34<54:23,  2.06it/s]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Training...:  91%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉                   | 55015/60729 [7:35:58<28:27,  3.35it/s]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Training...:  92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏               | 56000/60729 [7:43:46<38:13,  2.06it/s]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Training...:  94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌            | 57009/60729 [7:52:10<49:44,  1.25it/s]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Training...:  96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉         | 58023/60729 [8:00:17<05:25,  8.32it/s]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Training...:  97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏     | 59000/60729 [8:08:05<13:58,  2.06it/s]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Training...:  99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌  | 60000/60729 [8:16:28<05:53,  2.06it/s]
+Evaluating ...:   1%|█▊                                                                                                                                                                                                              | 4/462 [00:00<00:50,  9.12it/s]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Step... (60000 | Loss: 1.4433614015579224, Acc: 0.7172799110412598):   0%|                                                                                                                                                   | 0/3 [8:20:31<?, ?it/s]5:01,  2.06it/s]
+Traceback (most recent call last):
+  File "run_muril_bb_flax_mlm.py", line 695, in <module>
+    state, train_metric, dropout_rngs = p_train_step(state, model_inputs, dropout_rngs)
+  File "<string>", line 1, in __new__
+KeyboardInterrupt

wandb/run-20211107_122127-18mmtpt3/files/wandb-summary.json

@@ -1 +1 @@
-{"training_step": 50000, "learning_rate": 7.67694364185445e-05, "train_loss": 1.571732997894287, "_runtime": 24963, "_timestamp": 1636312650, "_step": 52, "eval_step": 45000, "eval_accuracy": 0.711020290851593, "eval_loss": 1.4846699237823486}
+{"training_step": 60000, "learning_rate": 7.096180343069136e-05, "train_loss": 1.5583317279815674, "_runtime": 30014, "_timestamp": 1636317701, "_step": 63, "eval_step": 60000, "eval_accuracy": 0.7172799110412598, "eval_loss": 1.4433614015579224, "_wandb": {"runtime": 30052}}

wandb/run-20211107_122127-18mmtpt3/logs/debug.log

@@ -23,3 +23,101 @@ config: {}
 2021-11-07 12:21:31,003 INFO    MainThread:13180 [wandb_run.py:_config_callback():911] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': True, 'do_eval': True, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 2, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 0.0001, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 3.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 10000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Nov07_12-21-20_t1v-n-ba7aa1a5-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 1000, 'logging_nan_inf_filter': True, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 50000, 'save_total_limit': 1, 'save_on_each_node': False, 'no_cuda': False, 'seed': 18, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'xpu_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 15000, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': 'en-ta-hi-te', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'HubStrategy.EVERY_SAVE', 'hub_token': None, 'gradient_checkpointing': False, 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': '', '_n_gpu': 0, '__cached__setup_devices': 'cpu'}
 2021-11-07 12:21:31,005 INFO    MainThread:13180 [wandb_run.py:_config_callback():911] config_cb None None {'model_name_or_path': 'muril-bigbird', 'model_type': None, 'config_name': None, 'tokenizer_name': 'muril-bigbird', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'bfloat16'}
 2021-11-07 12:21:31,006 INFO    MainThread:13180 [wandb_run.py:_config_callback():911] config_cb None None {'dataset_name': None, 'pretokenized': True, 'streaming': False, 'dataset_config_name': None, 'train_file': '/dev/shm/data-files', 'validation_file': '.', 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': None, 'mlm_probability': 0.15, 'pad_to_max_length': True, 'line_by_line': False}
+2021-11-07 20:42:18,919 INFO    MainThread:13180 [wandb_run.py:_atexit_cleanup():1663] got exitcode: 255
+2021-11-07 20:42:18,919 INFO    MainThread:13180 [wandb_run.py:_restore():1635] restore
+2021-11-07 20:42:21,203 INFO    MainThread:13180 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
+  wandb_count: 1
+}
+pusher_stats {
+  uploaded_bytes: 1755
+  total_bytes: 1755
+}
+
+2021-11-07 20:42:21,472 INFO    MainThread:13180 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
+  wandb_count: 1
+}
+pusher_stats {
+  uploaded_bytes: 1755
+  total_bytes: 1755
+}
+
+2021-11-07 20:42:21,632 INFO    MainThread:13180 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
+  wandb_count: 4
+}
+pusher_stats {
+  uploaded_bytes: 1755
+  total_bytes: 55503
+}
+
+2021-11-07 20:42:21,734 INFO    MainThread:13180 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 1755
+  total_bytes: 55779
+}
+
+2021-11-07 20:42:21,836 INFO    MainThread:13180 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 55779
+  total_bytes: 55779
+}
+
+2021-11-07 20:42:21,938 INFO    MainThread:13180 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 55779
+  total_bytes: 55779
+}
+
+2021-11-07 20:42:22,040 INFO    MainThread:13180 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 55779
+  total_bytes: 55779
+}
+
+2021-11-07 20:42:22,141 INFO    MainThread:13180 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 55779
+  total_bytes: 55779
+}
+
+2021-11-07 20:42:22,243 INFO    MainThread:13180 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 55779
+  total_bytes: 55779
+}
+
+2021-11-07 20:42:22,612 INFO    MainThread:13180 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 55779
+  total_bytes: 55779
+}
+
+2021-11-07 20:42:22,963 INFO    MainThread:13180 [wandb_run.py:_wait_for_finish():1793] got exit ret: done: true
+exit_result {
+}
+file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 55779
+  total_bytes: 55779
+}
+local_info {
+}
+
+2021-11-07 20:42:24,403 INFO    MainThread:13180 [wandb_run.py:_append_history():2011] rendering history
+2021-11-07 20:42:24,404 INFO    MainThread:13180 [wandb_run.py:_append_summary():1966] rendering summary
+2021-11-07 20:42:24,404 INFO    MainThread:13180 [wandb_run.py:_append_files():2061] logging synced files

wandb/run-20211107_204808-2ns3lo5n/files/config.yaml

@@ -0,0 +1,341 @@
+wandb_version: 1
+
+__cached__setup_devices:
+  desc: null
+  value: cpu
+_n_gpu:
+  desc: null
+  value: 0
+_wandb:
+  desc: null
+  value:
+    cli_version: 0.12.6
+    framework: huggingface
+    huggingface_version: 4.13.0.dev0
+    is_jupyter_run: false
+    is_kaggle_kernel: true
+    python_version: 3.8.10
+    start_time: 1636318088
+    t:
+      1:
+      - 1
+      - 2
+      - 3
+      - 11
+      - 12
+      3:
+      - 13
+      4: 3.8.10
+      5: 0.12.6
+      6: 4.13.0.dev0
+      8:
+      - 2
+      - 5
+adafactor:
+  desc: null
+  value: false
+adam_beta1:
+  desc: null
+  value: 0.9
+adam_beta2:
+  desc: null
+  value: 0.999
+adam_epsilon:
+  desc: null
+  value: 1.0e-08
+cache_dir:
+  desc: null
+  value: null
+config_name:
+  desc: null
+  value: null
+dataloader_drop_last:
+  desc: null
+  value: false
+dataloader_num_workers:
+  desc: null
+  value: 0
+dataloader_pin_memory:
+  desc: null
+  value: true
+dataset_config_name:
+  desc: null
+  value: null
+dataset_name:
+  desc: null
+  value: null
+ddp_find_unused_parameters:
+  desc: null
+  value: null
+debug:
+  desc: null
+  value: []
+deepspeed:
+  desc: null
+  value: null
+disable_tqdm:
+  desc: null
+  value: false
+do_eval:
+  desc: null
+  value: true
+do_predict:
+  desc: null
+  value: false
+do_train:
+  desc: null
+  value: true
+dtype:
+  desc: null
+  value: bfloat16
+eval_accumulation_steps:
+  desc: null
+  value: null
+eval_steps:
+  desc: null
+  value: 5000
+evaluation_strategy:
+  desc: null
+  value: IntervalStrategy.NO
+fp16:
+  desc: null
+  value: false
+fp16_backend:
+  desc: null
+  value: auto
+fp16_full_eval:
+  desc: null
+  value: false
+fp16_opt_level:
+  desc: null
+  value: O1
+gradient_accumulation_steps:
+  desc: null
+  value: 1
+gradient_checkpointing:
+  desc: null
+  value: false
+greater_is_better:
+  desc: null
+  value: null
+group_by_length:
+  desc: null
+  value: false
+hub_model_id:
+  desc: null
+  value: null
+hub_strategy:
+  desc: null
+  value: HubStrategy.EVERY_SAVE
+hub_token:
+  desc: null
+  value: null
+ignore_data_skip:
+  desc: null
+  value: false
+label_names:
+  desc: null
+  value: null
+label_smoothing_factor:
+  desc: null
+  value: 0.0
+learning_rate:
+  desc: null
+  value: 5.0e-05
+length_column_name:
+  desc: null
+  value: length
+line_by_line:
+  desc: null
+  value: false
+load_best_model_at_end:
+  desc: null
+  value: false
+local_rank:
+  desc: null
+  value: -1
+log_level:
+  desc: null
+  value: -1
+log_level_replica:
+  desc: null
+  value: -1
+log_on_each_node:
+  desc: null
+  value: true
+logging_dir:
+  desc: null
+  value: ./runs/Nov07_20-47-12_t1v-n-ba7aa1a5-w-0
+logging_first_step:
+  desc: null
+  value: false
+logging_nan_inf_filter:
+  desc: null
+  value: true
+logging_steps:
+  desc: null
+  value: 500
+logging_strategy:
+  desc: null
+  value: IntervalStrategy.STEPS
+lr_scheduler_type:
+  desc: null
+  value: SchedulerType.LINEAR
+max_grad_norm:
+  desc: null
+  value: 1.0
+max_seq_length:
+  desc: null
+  value: 4096
+max_steps:
+  desc: null
+  value: -1
+metric_for_best_model:
+  desc: null
+  value: null
+mlm_probability:
+  desc: null
+  value: 0.15
+model_name_or_path:
+  desc: null
+  value: muril-bigbird
+model_type:
+  desc: null
+  value: null
+mp_parameters:
+  desc: null
+  value: ''
+no_cuda:
+  desc: null
+  value: false
+num_train_epochs:
+  desc: null
+  value: 3.0
+output_dir:
+  desc: null
+  value: ./
+overwrite_cache:
+  desc: null
+  value: false
+overwrite_output_dir:
+  desc: null
+  value: true
+pad_to_max_length:
+  desc: null
+  value: true
+past_index:
+  desc: null
+  value: -1
+per_device_eval_batch_size:
+  desc: null
+  value: 2
+per_device_train_batch_size:
+  desc: null
+  value: 1
+per_gpu_eval_batch_size:
+  desc: null
+  value: null
+per_gpu_train_batch_size:
+  desc: null
+  value: null
+prediction_loss_only:
+  desc: null
+  value: false
+preprocessing_num_workers:
+  desc: null
+  value: null
+pretokenized:
+  desc: null
+  value: true
+push_to_hub:
+  desc: null
+  value: true
+push_to_hub_model_id:
+  desc: null
+  value: null
+push_to_hub_organization:
+  desc: null
+  value: null
+push_to_hub_token:
+  desc: null
+  value: null
+remove_unused_columns:
+  desc: null
+  value: true
+report_to:
+  desc: null
+  value:
+  - tensorboard
+  - wandb
+resume_from_checkpoint:
+  desc: null
+  value: null
+run_name:
+  desc: null
+  value: en-ta-hi-te-from-50k
+save_on_each_node:
+  desc: null
+  value: false
+save_steps:
+  desc: null
+  value: 50000
+save_strategy:
+  desc: null
+  value: IntervalStrategy.STEPS
+save_total_limit:
+  desc: null
+  value: 1
+seed:
+  desc: null
+  value: 18
+sharded_ddp:
+  desc: null
+  value: []
+skip_memory_metrics:
+  desc: null
+  value: true
+streaming:
+  desc: null
+  value: false
+tokenizer_name:
+  desc: null
+  value: muril-bigbird
+tpu_metrics_debug:
+  desc: null
+  value: false
+tpu_num_cores:
+  desc: null
+  value: null
+train_file:
+  desc: null
+  value: /dev/shm/data-files
+train_ref_file:
+  desc: null
+  value: null
+use_fast_tokenizer:
+  desc: null
+  value: true
+use_legacy_prediction_loop:
+  desc: null
+  value: false
+validation_file:
+  desc: null
+  value: .
+validation_ref_file:
+  desc: null
+  value: null
+validation_split_percentage:
+  desc: null
+  value: 5
+warmup_ratio:
+  desc: null
+  value: 0.0
+warmup_steps:
+  desc: null
+  value: 0
+weight_decay:
+  desc: null
+  value: 0.01
+xpu_backend:
+  desc: null
+  value: null

wandb/run-20211107_204808-2ns3lo5n/files/output.log

@@ -0,0 +1,98 @@
+[20:48:27] - INFO - absl - A polynomial schedule was set with a non-positive `transition_steps` value; this results in a constant schedule with value `init_value`.
+Epoch ... (1/3):   0%|                                                                                                                                                                                                         | 0/3 [00:00<?, ?it/s]
+Training...:   0%|                                                                                                                                                                                                         | 0/61250 [00:00<?, ?it/s]
+Epoch ... (1/3):   0%|                                                                                                                                                                                                         | 0/3 [03:12<?, ?it/s]
+Training...:   0%|                                                                                                                                                                                                         | 0/61250 [02:05<?, ?it/s]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+tcmalloc: large alloc 1212121088 bytes == 0x3920c6000 @  0x7fee676b5680 0x7fee676d5bdd 0x7fecb83f720d 0x7fecb8404069 0x7fecb840556e 0x7fecb8400bd3 0x7fecb84011fe 0x5042c6 0x56bf09 0x56a0ba 0x5f6343 0x56cf2a 0x5f6166 0x56bddd 0x7fecb83f7410 0x7fecb83f867a 0x7fecb83fa6f7 0x7fecb84042a3 0x7fecb8404e87 0x7fecb8404e87 0x7fecb8404e87 0x7fecb8404e87 0x7fecb8404e87 0x7fecb8400bd3 0x7fecb84011fe 0x5042c6 0x56bf09 0x56a0ba 0x5f6343 0x56cf2a 0x56a0ba
+tcmalloc: large alloc 1237311488 bytes == 0x3fe6ba000 @  0x7fee676b5680 0x7fee676d5bdd 0x7fecb83f720d 0x7fecb8405340 0x7fecb8404e87 0x7fecb8404e87 0x7fecb8404e87 0x7fecb8404e87 0x7fecb8404e87 0x7fecb8400bd3 0x7fecb84011fe 0x5042c6 0x56bf09 0x56a0ba 0x5f6343 0x56cf2a 0x56a0ba 0x5f6343 0x56cf2a 0x5f6166 0x570e46 0x56a0ba 0x5f6343 0x56cf2a 0x56a0ba 0x68d5b7 0x67cd01 0x67cd7f 0x67ce21 0x67ef47 0x6b7242
+tcmalloc: large alloc 3161391104 bytes == 0x4482b8000 @  0x7fee676b5680 0x7fee676d5bdd 0x7fecb83f720d 0x7fecb8405340 0x7fecb8404e87 0x7fecb8404e87 0x7fecb8404e87 0x7fecb8404e87 0x7fecb8404e87 0x7fecb8404e87 0x7fecb8404e87 0x7fecb8400bd3 0x7fecb84011fe 0x5042c6 0x56bf09 0x56a0ba 0x5f6343 0x56cf2a 0x56a0ba 0x5f6343 0x56cf2a 0x5f6166 0x570e46 0x56a0ba 0x5f6343 0x56cf2a 0x56a0ba 0x68d5b7 0x67cd01 0x67cd7f 0x67ce21
+tcmalloc: large alloc 2886115328 bytes == 0x3920c6000 @  0x7fee676b5680 0x7fee676d6824 0x5f9f21 0x7fecb8400c6f 0x7fecb84011fe 0x5042c6 0x56bf09 0x56a0ba 0x5f6343 0x56cf2a 0x56a0ba 0x5f6343 0x56cf2a 0x5f6166 0x570e46 0x56a0ba 0x5f6343 0x56cf2a 0x56a0ba 0x68d5b7 0x67cd01 0x67cd7f 0x67ce21 0x67ef47 0x6b7242 0x6b75cd 0x7fee674cc0b3 0x5fb18e

wandb/run-20211107_204808-2ns3lo5n/files/requirements.txt

@@ -0,0 +1,101 @@
+absl-py==0.15.0
+aiohttp==3.8.0
+aiosignal==1.2.0
+astunparse==1.6.3
+async-timeout==4.0.0
+attrs==21.2.0
+cachetools==4.2.4
+certifi==2021.10.8
+charset-normalizer==2.0.7
+chex==0.0.8
+click==8.0.3
+configparser==5.1.0
+cycler==0.11.0
+datasets==1.15.1
+dill==0.3.4
+dm-tree==0.1.6
+docker-pycreds==0.4.0
+filelock==3.3.2
+flatbuffers==2.0
+flax==0.3.6
+frozenlist==1.2.0
+fsspec==2021.11.0
+gast==0.4.0
+gitdb==4.0.9
+gitpython==3.1.24
+google-auth-oauthlib==0.4.6
+google-auth==2.3.3
+google-pasta==0.2.0
+grpcio==1.41.1
+h5py==3.5.0
+huggingface-hub==0.1.1
+idna==3.3
+jax==0.2.24
+jaxlib==0.1.73
+joblib==1.1.0
+kaggle==1.5.12
+keras-preprocessing==1.1.2
+keras==2.7.0
+kiwisolver==1.3.2
+libclang==12.0.0
+libtpu-nightly==0.1.dev20211018
+markdown==3.3.4
+matplotlib==3.4.3
+msgpack==1.0.2
+multidict==5.2.0
+multiprocess==0.70.12.2
+numpy==1.21.4
+oauthlib==3.1.1
+opt-einsum==3.3.0
+optax==0.0.9
+packaging==21.2
+pandas==1.3.4
+pathtools==0.1.2
+pillow==8.4.0
+pip==21.3.1
+pkg-resources==0.0.0
+promise==2.3
+protobuf==3.19.1
+psutil==5.8.0
+pyarrow==6.0.0
+pyasn1-modules==0.2.8
+pyasn1==0.4.8
+pyparsing==2.4.7
+python-dateutil==2.8.2
+python-slugify==5.0.2
+pytz==2021.3
+pyyaml==6.0
+regex==2021.11.2
+requests-oauthlib==1.3.0
+requests==2.26.0
+rsa==4.7.2
+sacremoses==0.0.46
+scipy==1.7.1
+sentry-sdk==1.4.3
+setuptools==44.0.0
+shortuuid==1.0.1
+six==1.16.0
+smmap==5.0.0
+subprocess32==3.5.4
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.0
+tensorboard==2.7.0
+tensorflow-estimator==2.7.0
+tensorflow-io-gcs-filesystem==0.21.0
+tensorflow==2.7.0
+termcolor==1.1.0
+text-unidecode==1.3
+tokenizers==0.10.3
+toolz==0.11.1
+torch==1.10.0
+tqdm==4.62.3
+transformers==4.13.0.dev0
+typing-extensions==3.10.0.2
+urllib3==1.26.7
+wandb==0.12.6
+werkzeug==2.0.2
+wheel==0.37.0
+wrapt==1.13.3
+xxhash==2.0.2
+yarl==1.7.2
+yaspin==2.1.0

wandb/run-20211107_204808-2ns3lo5n/files/wandb-metadata.json

@@ -0,0 +1,67 @@
+{
+    "os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29",
+    "python": "3.8.10",
+    "heartbeatAt": "2021-11-07T20:48:11.579826",
+    "startedAt": "2021-11-07T20:48:08.404632",
+    "docker": null,
+    "cpu_count": 96,
+    "cuda": null,
+    "args": [
+        "--model_name_or_path",
+        "muril-bigbird",
+        "--tokenizer_name",
+        "muril-bigbird",
+        "--dtype",
+        "bfloat16",
+        "--pretokenized",
+        "--train_file",
+        "/dev/shm/data-files",
+        "--validation_file",
+        ".",
+        "--max_seq_length",
+        "4096",
+        "--pad_to_max_length",
+        "--output_dir",
+        "./",
+        "--overwrite_output_dir",
+        "--do_train",
+        "--do_eval",
+        "--eval_steps",
+        "5000",
+        "--per_device_train_batch_size",
+        "1",
+        "--per_device_eval_batch_size",
+        "2",
+        "--learning_rate",
+        "5e-5",
+        "--weight_decay",
+        "0.01",
+        "--num_train_epochs",
+        "3",
+        "--warmup_steps",
+        "0",
+        "--logging_steps",
+        "500",
+        "--save_steps",
+        "50000",
+        "--save_total_limit",
+        "1",
+        "--seed",
+        "18",
+        "--push_to_hub",
+        "--run_name",
+        "en-ta-hi-te-from-50k"
+    ],
+    "state": "running",
+    "program": "run_muril_bb_flax_mlm.py",
+    "codePath": "run_muril_bb_flax_mlm.py",
+    "git": {
+        "remote": "https://huggingface.co/nbroad/muril-bigbird-base-1mil",
+        "commit": "52d9b405c9e32ad5079f76867da58f0d64320170"
+    },
+    "email": "nicholas@nmbroad.com",
+    "root": "/home/nicholasbroad/muril-bigbird-base-1mil",
+    "host": "t1v-n-ba7aa1a5-w-0",
+    "username": "nicholasbroad",
+    "executable": "/home/nicholasbroad/venv/bin/python"
+}

wandb/run-20211107_204808-2ns3lo5n/files/wandb-summary.json

@@ -0,0 +1 @@
+{"training_step": 50000, "learning_rate": 4.999999873689376e-05, "train_loss": 1.9627385139465332, "_runtime": 422, "_timestamp": 1636318510, "_step": 1, "eval_step": 50000, "eval_accuracy": 0.6992624998092651, "eval_loss": 1.5497748851776123}

wandb/run-20211107_204808-2ns3lo5n/logs/debug-internal.log

@@ -0,0 +1,226 @@
+2021-11-07 20:48:09,073 INFO    MainThread:21539 [internal.py:wandb_internal():87] W&B internal server running at pid: 21539, started at: 2021-11-07 20:48:09.073384
+2021-11-07 20:48:09,075 DEBUG   HandlerThread:21539 [handler.py:handle_request():130] handle_request: check_version
+2021-11-07 20:48:09,075 INFO    WriterThread:21539 [datastore.py:open_for_write():77] open: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/run-2ns3lo5n.wandb
+2021-11-07 20:48:09,077 DEBUG   SenderThread:21539 [sender.py:send():238] send: header
+2021-11-07 20:48:09,077 DEBUG   SenderThread:21539 [sender.py:send_request():252] send_request: check_version
+2021-11-07 20:48:09,123 DEBUG   SenderThread:21539 [sender.py:send():238] send: run
+2021-11-07 20:48:09,499 INFO    SenderThread:21539 [dir_watcher.py:__init__():169] watching files in: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files
+2021-11-07 20:48:09,499 INFO    SenderThread:21539 [sender.py:_start_run_threads():825] run started: 2ns3lo5n with start time 1636318088
+2021-11-07 20:48:09,500 DEBUG   SenderThread:21539 [sender.py:send():238] send: summary
+2021-11-07 20:48:09,500 INFO    SenderThread:21539 [sender.py:_save_file():960] saving file wandb-summary.json with policy end
+2021-11-07 20:48:09,500 DEBUG   HandlerThread:21539 [handler.py:handle_request():130] handle_request: run_start
+2021-11-07 20:48:10,502 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_created():217] file/dir created: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/wandb-summary.json
+2021-11-07 20:48:11,579 DEBUG   HandlerThread:21539 [meta.py:__init__():40] meta init
+2021-11-07 20:48:11,579 DEBUG   HandlerThread:21539 [meta.py:__init__():54] meta init done
+2021-11-07 20:48:11,579 DEBUG   HandlerThread:21539 [meta.py:probe():214] probe
+2021-11-07 20:48:11,580 DEBUG   HandlerThread:21539 [meta.py:_setup_git():204] setup git
+2021-11-07 20:48:11,611 DEBUG   HandlerThread:21539 [meta.py:_setup_git():211] setup git done
+2021-11-07 20:48:11,611 DEBUG   HandlerThread:21539 [meta.py:_save_pip():58] save pip
+2021-11-07 20:48:11,612 DEBUG   HandlerThread:21539 [meta.py:_save_pip():72] save pip done
+2021-11-07 20:48:11,612 DEBUG   HandlerThread:21539 [meta.py:probe():252] probe done
+2021-11-07 20:48:11,615 DEBUG   SenderThread:21539 [sender.py:send():238] send: files
+2021-11-07 20:48:11,615 INFO    SenderThread:21539 [sender.py:_save_file():960] saving file wandb-metadata.json with policy now
+2021-11-07 20:48:11,620 DEBUG   HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
+2021-11-07 20:48:11,621 DEBUG   SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
+2021-11-07 20:48:11,755 DEBUG   SenderThread:21539 [sender.py:send():238] send: config
+2021-11-07 20:48:11,756 DEBUG   SenderThread:21539 [sender.py:send():238] send: config
+2021-11-07 20:48:11,756 DEBUG   SenderThread:21539 [sender.py:send():238] send: config
+2021-11-07 20:48:12,094 INFO    Thread-11 :21539 [upload_job.py:push():137] Uploaded file /tmp/tmpzv74dks4wandb/1kin4cdo-wandb-metadata.json
+2021-11-07 20:48:12,504 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_created():217] file/dir created: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:48:12,505 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_created():217] file/dir created: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/requirements.txt
+2021-11-07 20:48:12,505 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_created():217] file/dir created: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/wandb-metadata.json
+2021-11-07 20:48:27,125 DEBUG   HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
+2021-11-07 20:48:27,126 DEBUG   SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
+2021-11-07 20:48:30,511 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:48:39,514 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/config.yaml
+2021-11-07 20:48:39,657 DEBUG   SenderThread:21539 [sender.py:send():238] send: stats
+2021-11-07 20:48:42,263 DEBUG   HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
+2021-11-07 20:48:42,263 DEBUG   SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
+2021-11-07 20:48:57,395 DEBUG   HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
+2021-11-07 20:48:57,395 DEBUG   SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
+2021-11-07 20:49:09,716 DEBUG   SenderThread:21539 [sender.py:send():238] send: stats
+2021-11-07 20:49:12,528 DEBUG   HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
+2021-11-07 20:49:12,528 DEBUG   SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
+2021-11-07 20:49:27,664 DEBUG   HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
+2021-11-07 20:49:27,665 DEBUG   SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
+2021-11-07 20:49:36,535 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:49:39,780 DEBUG   SenderThread:21539 [sender.py:send():238] send: stats
+2021-11-07 20:49:42,824 DEBUG   HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
+2021-11-07 20:49:42,824 DEBUG   SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
+2021-11-07 20:49:57,982 DEBUG   HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
+2021-11-07 20:49:57,982 DEBUG   SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
+2021-11-07 20:50:09,846 DEBUG   SenderThread:21539 [sender.py:send():238] send: stats
+2021-11-07 20:50:13,120 DEBUG   HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
+2021-11-07 20:50:13,120 DEBUG   SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
+2021-11-07 20:50:28,253 DEBUG   HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
+2021-11-07 20:50:28,253 DEBUG   SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
+2021-11-07 20:50:39,916 DEBUG   SenderThread:21539 [sender.py:send():238] send: stats
+2021-11-07 20:50:43,387 DEBUG   HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
+2021-11-07 20:50:43,387 DEBUG   SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
+2021-11-07 20:50:58,521 DEBUG   HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
+2021-11-07 20:50:58,522 DEBUG   SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
+2021-11-07 20:51:09,989 DEBUG   SenderThread:21539 [sender.py:send():238] send: stats
+2021-11-07 20:51:13,656 DEBUG   HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
+2021-11-07 20:51:13,656 DEBUG   SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
+2021-11-07 20:51:28,789 DEBUG   HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
+2021-11-07 20:51:28,790 DEBUG   SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
+2021-11-07 20:51:40,065 DEBUG   SenderThread:21539 [sender.py:send():238] send: stats
+2021-11-07 20:51:40,800 DEBUG   SenderThread:21539 [sender.py:send():238] send: history
+2021-11-07 20:51:40,801 DEBUG   SenderThread:21539 [sender.py:send():238] send: summary
+2021-11-07 20:51:40,801 INFO    SenderThread:21539 [sender.py:_save_file():960] saving file wandb-summary.json with policy end
+2021-11-07 20:51:41,580 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/wandb-summary.json
+2021-11-07 20:51:42,580 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:51:43,934 DEBUG   HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
+2021-11-07 20:51:43,934 DEBUG   SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
+2021-11-07 20:51:59,097 DEBUG   HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
+2021-11-07 20:51:59,097 DEBUG   SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
+2021-11-07 20:52:04,589 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:52:06,590 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:52:08,591 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:52:10,142 DEBUG   SenderThread:21539 [sender.py:send():238] send: stats
+2021-11-07 20:52:10,592 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:52:12,593 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:52:14,272 DEBUG   HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
+2021-11-07 20:52:14,272 DEBUG   SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
+2021-11-07 20:52:14,593 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:52:16,595 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:52:18,595 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:52:20,596 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:52:22,597 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:52:24,598 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:52:26,599 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:52:28,599 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:52:29,407 DEBUG   HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
+2021-11-07 20:52:29,408 DEBUG   SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
+2021-11-07 20:52:30,600 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:52:32,601 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:52:34,602 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:52:36,603 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:52:38,604 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:52:40,215 DEBUG   SenderThread:21539 [sender.py:send():238] send: stats
+2021-11-07 20:52:40,604 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:52:42,606 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:52:44,538 DEBUG   HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
+2021-11-07 20:52:44,538 DEBUG   SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
+2021-11-07 20:52:45,607 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:52:47,608 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:52:49,609 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:52:51,610 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:52:53,611 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:52:55,612 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:52:57,613 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:52:59,614 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:52:59,744 DEBUG   HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
+2021-11-07 20:52:59,744 DEBUG   SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
+2021-11-07 20:53:01,615 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:53:03,615 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:53:05,616 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:53:07,617 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:53:09,618 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:53:10,287 DEBUG   SenderThread:21539 [sender.py:send():238] send: stats
+2021-11-07 20:53:11,619 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:53:13,620 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:53:14,884 DEBUG   HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
+2021-11-07 20:53:14,884 DEBUG   SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
+2021-11-07 20:53:15,621 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:53:17,621 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:53:19,622 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:53:21,623 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:53:23,624 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:53:25,625 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:53:27,626 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:53:29,626 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:53:30,028 DEBUG   HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
+2021-11-07 20:53:30,028 DEBUG   SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
+2021-11-07 20:53:31,627 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:53:33,628 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:53:35,629 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:53:37,630 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:53:39,631 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:53:40,360 DEBUG   SenderThread:21539 [sender.py:send():238] send: stats
+2021-11-07 20:53:41,632 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:53:43,633 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:53:45,199 DEBUG   HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
+2021-11-07 20:53:45,199 DEBUG   SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
+2021-11-07 20:53:45,634 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:53:47,635 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:53:49,636 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:53:51,636 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:53:53,637 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:53:55,638 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:53:57,639 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:53:59,640 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:54:00,332 DEBUG   HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
+2021-11-07 20:54:00,332 DEBUG   SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
+2021-11-07 20:54:01,641 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:54:03,642 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:54:05,643 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:54:07,644 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:54:09,644 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:54:10,433 DEBUG   SenderThread:21539 [sender.py:send():238] send: stats
+2021-11-07 20:54:11,645 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:54:13,646 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:54:15,474 DEBUG   HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
+2021-11-07 20:54:15,474 DEBUG   SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
+2021-11-07 20:54:15,647 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:54:17,647 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:54:19,648 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:54:21,649 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:54:23,650 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:54:25,651 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:54:27,651 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:54:29,652 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:54:30,676 DEBUG   HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
+2021-11-07 20:54:30,677 DEBUG   SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
+2021-11-07 20:54:31,653 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:54:34,654 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:54:36,655 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:54:38,656 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:54:40,505 DEBUG   SenderThread:21539 [sender.py:send():238] send: stats
+2021-11-07 20:54:40,657 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:54:42,657 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:54:44,658 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:54:45,808 DEBUG   HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
+2021-11-07 20:54:45,809 DEBUG   SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
+2021-11-07 20:54:46,659 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:54:48,660 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:54:50,661 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:54:52,662 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:54:54,663 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:54:56,664 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:54:58,664 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:55:00,665 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:55:00,946 DEBUG   HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
+2021-11-07 20:55:00,947 DEBUG   SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
+2021-11-07 20:55:02,666 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:55:10,442 DEBUG   SenderThread:21539 [sender.py:send():238] send: history
+2021-11-07 20:55:10,443 DEBUG   SenderThread:21539 [sender.py:send():238] send: summary
+2021-11-07 20:55:10,445 INFO    SenderThread:21539 [sender.py:_save_file():960] saving file wandb-summary.json with policy end
+2021-11-07 20:55:10,577 DEBUG   SenderThread:21539 [sender.py:send():238] send: stats
+2021-11-07 20:55:10,669 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/wandb-summary.json
+2021-11-07 20:55:17,590 DEBUG   HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
+2021-11-07 20:55:17,591 DEBUG   SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
+2021-11-07 20:55:17,672 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:55:21,674 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:55:23,675 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:55:32,740 DEBUG   HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
+2021-11-07 20:55:32,740 DEBUG   SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
+2021-11-07 20:55:40,655 DEBUG   SenderThread:21539 [sender.py:send():238] send: stats
+2021-11-07 20:55:47,874 DEBUG   HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
+2021-11-07 20:55:47,875 DEBUG   SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
+2021-11-07 20:55:49,684 INFO    Thread-8  :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
+2021-11-07 20:56:03,011 DEBUG   HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
+2021-11-07 20:56:03,012 DEBUG   SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
+2021-11-07 20:56:10,732 DEBUG   SenderThread:21539 [sender.py:send():238] send: stats
+2021-11-07 20:56:18,145 DEBUG   HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
+2021-11-07 20:56:18,145 DEBUG   SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
+2021-11-07 20:56:33,278 DEBUG   HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
+2021-11-07 20:56:33,278 DEBUG   SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
+2021-11-07 20:56:40,805 DEBUG   SenderThread:21539 [sender.py:send():238] send: stats
+2021-11-07 20:56:48,418 DEBUG   HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
+2021-11-07 20:56:48,419 DEBUG   SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
+2021-11-07 20:57:03,562 DEBUG   HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
+2021-11-07 20:57:03,563 DEBUG   SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
+2021-11-07 20:57:10,879 DEBUG   SenderThread:21539 [sender.py:send():238] send: stats
+2021-11-07 20:57:18,693 DEBUG   HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
+2021-11-07 20:57:18,694 DEBUG   SenderThread:21539 [sender.py:send_request():252] send_request: stop_status

wandb/run-20211107_204808-2ns3lo5n/logs/debug.log

@@ -0,0 +1,25 @@
+2021-11-07 20:48:08,405 INFO    MainThread:20231 [wandb_setup.py:_flush():71] setting env: {}
+2021-11-07 20:48:08,405 INFO    MainThread:20231 [wandb_setup.py:_flush():71] setting login settings: {}
+2021-11-07 20:48:08,405 INFO    MainThread:20231 [wandb_init.py:_log_setup():357] Logging user logs to /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/logs/debug.log
+2021-11-07 20:48:08,406 INFO    MainThread:20231 [wandb_init.py:_log_setup():358] Logging internal logs to /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/logs/debug-internal.log
+2021-11-07 20:48:08,406 INFO    MainThread:20231 [wandb_init.py:init():390] calling init triggers
+2021-11-07 20:48:08,406 INFO    MainThread:20231 [wandb_init.py:init():395] wandb.init called with sweep_config: {}
+config: {}
+2021-11-07 20:48:08,406 INFO    MainThread:20231 [wandb_init.py:init():435] starting backend
+2021-11-07 20:48:08,406 INFO    MainThread:20231 [backend.py:_multiprocessing_setup():95] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2021-11-07 20:48:08,444 INFO    MainThread:20231 [backend.py:ensure_launched():199] starting backend process...
+2021-11-07 20:48:08,481 INFO    MainThread:20231 [backend.py:ensure_launched():204] started backend process with pid: 21539
+2021-11-07 20:48:08,483 INFO    MainThread:20231 [wandb_init.py:init():444] backend started and connected
+2021-11-07 20:48:08,486 INFO    MainThread:20231 [wandb_init.py:init():503] updated telemetry
+2021-11-07 20:48:08,568 INFO    MainThread:20231 [wandb_init.py:init():533] communicating current version
+2021-11-07 20:48:09,122 INFO    MainThread:20231 [wandb_init.py:init():538] got version response 
+2021-11-07 20:48:09,122 INFO    MainThread:20231 [wandb_init.py:init():548] communicating run to backend with 30 second timeout
+2021-11-07 20:48:09,500 INFO    MainThread:20231 [wandb_init.py:init():576] starting run threads in backend
+2021-11-07 20:48:11,620 INFO    MainThread:20231 [wandb_run.py:_console_start():1693] atexit reg
+2021-11-07 20:48:11,620 INFO    MainThread:20231 [wandb_run.py:_redirect():1567] redirect: SettingsConsole.REDIRECT
+2021-11-07 20:48:11,621 INFO    MainThread:20231 [wandb_run.py:_redirect():1572] Redirecting console.
+2021-11-07 20:48:11,622 INFO    MainThread:20231 [wandb_run.py:_redirect():1628] Redirects installed.
+2021-11-07 20:48:11,622 INFO    MainThread:20231 [wandb_init.py:init():603] run started, returning control to user process
+2021-11-07 20:48:11,628 INFO    MainThread:20231 [wandb_run.py:_config_callback():911] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': True, 'do_eval': True, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 2, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 5e-05, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 3.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 0, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Nov07_20-47-12_t1v-n-ba7aa1a5-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 500, 'logging_nan_inf_filter': True, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 50000, 'save_total_limit': 1, 'save_on_each_node': False, 'no_cuda': False, 'seed': 18, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'xpu_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 5000, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': 'en-ta-hi-te-from-50k', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'HubStrategy.EVERY_SAVE', 'hub_token': None, 'gradient_checkpointing': False, 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': '', '_n_gpu': 0, '__cached__setup_devices': 'cpu'}
+2021-11-07 20:48:11,630 INFO    MainThread:20231 [wandb_run.py:_config_callback():911] config_cb None None {'model_name_or_path': 'muril-bigbird', 'model_type': None, 'config_name': None, 'tokenizer_name': 'muril-bigbird', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'bfloat16'}
+2021-11-07 20:48:11,631 INFO    MainThread:20231 [wandb_run.py:_config_callback():911] config_cb None None {'dataset_name': None, 'pretokenized': True, 'streaming': False, 'dataset_config_name': None, 'train_file': '/dev/shm/data-files', 'validation_file': '.', 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': None, 'mlm_probability': 0.15, 'pad_to_max_length': True, 'line_by_line': False}