nbroad commited on
Commit
fc39d0b
·
1 Parent(s): 0c3c680

Saving weights and logs of step 125000

Browse files
.gitattributes CHANGED
@@ -25,4 +25,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
25
  *.zip filter=lfs diff=lfs merge=lfs -text
26
  *.zstandard filter=lfs diff=lfs merge=lfs -text
27
  *tfevents* filter=lfs diff=lfs merge=lfs -text
28
- checkpoint_100000 filter=lfs diff=lfs merge=lfs -text
 
25
  *.zip filter=lfs diff=lfs merge=lfs -text
26
  *.zstandard filter=lfs diff=lfs merge=lfs -text
27
  *tfevents* filter=lfs diff=lfs merge=lfs -text
28
+ checkpoint_125000 filter=lfs diff=lfs merge=lfs -text
150k_flax_model.msgpack ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ee88317b8117ad2298f4690bf5c4986bdc07f35f656802d692d990250b6708cd
3
+ size 962037499
checkpoint_100000 → checkpoint_125000 RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:607b75832098cb034fbbfc2f47553b907eec9f76d0be917f739c28406327da65
3
  size 2886112612
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a8f6b7575b57b3284af5ced0bfe2ea8ce8b959173ed5573997121705249251bb
3
  size 2886112612
events.out.tfevents.1636359665.t1v-n-ba7aa1a5-w-0.3923.0.v2 CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:f3e435bb2c65f3d4074def02fbf055024935707dc0f794440ea8738969b3f679
3
- size 7454960
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e9802bbfef48a6fd0d14ca75eab07888ea1b0720edd2384f4714b3269c0ec3f3
3
+ size 7604030
events.out.tfevents.1636387302.t1v-n-ba7aa1a5-w-0.10728.0.v2 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:38f93d148d7cabe02f29e4528599211aa4891a5c5a7ebe669ceefdba0eadab9f
3
+ size 3727500
flax_model.msgpack CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:ee88317b8117ad2298f4690bf5c4986bdc07f35f656802d692d990250b6708cd
3
  size 962037499
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cd3ae47e8681fcf1792ce542b31355ef2c7e3c2743132d86f4cd6f78787a237c
3
  size 962037499
muril-bigbird/flax_model.msgpack CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:9a27a51818882c80dc60c00a6758070d78bd85e45a66c91a5c31cf6f491516c7
3
  size 962037499
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ee88317b8117ad2298f4690bf5c4986bdc07f35f656802d692d990250b6708cd
3
  size 962037499
run_muril_bb_flax.sh CHANGED
@@ -26,4 +26,4 @@ python run_muril_bb_flax_mlm.py \
26
  --save_total_limit 1 \
27
  --seed 18 \
28
  --push_to_hub \
29
- --run_name "en-ta-hi-te-from-100k"
 
26
  --save_total_limit 1 \
27
  --seed 18 \
28
  --push_to_hub \
29
+ --run_name "en-ta-hi-te-from-150k"
run_muril_bb_flax_mlm.py CHANGED
@@ -343,7 +343,7 @@ if __name__ == "__main__":
343
  files = p.glob("*/*.parquet")
344
  files = [str(x) for x in files]
345
  random.shuffle(files)
346
- files = files[100:150]
347
  datasets= load_dataset("parquet", data_files=files, split="train", cache_dir="/dev/shm")
348
  datasets = datasets.shuffle(seed=training_args.seed)
349
  datasets = datasets.train_test_split(test_size = 0.02, seed=training_args.seed)
 
343
  files = p.glob("*/*.parquet")
344
  files = [str(x) for x in files]
345
  random.shuffle(files)
346
+ files = files[150:200]
347
  datasets= load_dataset("parquet", data_files=files, split="train", cache_dir="/dev/shm")
348
  datasets = datasets.shuffle(seed=training_args.seed)
349
  datasets = datasets.train_test_split(test_size = 0.02, seed=training_args.seed)
wandb/debug-internal.log CHANGED
@@ -1 +1 @@
1
- run-20211108_082107-2idnuxcc/logs/debug-internal.log
 
1
+ run-20211108_160143-33jzyy4p/logs/debug-internal.log
wandb/debug.log CHANGED
@@ -1 +1 @@
1
- run-20211108_082107-2idnuxcc/logs/debug.log
 
1
+ run-20211108_160143-33jzyy4p/logs/debug.log
wandb/latest-run CHANGED
@@ -1 +1 @@
1
- run-20211108_082107-2idnuxcc
 
1
+ run-20211108_160143-33jzyy4p
wandb/run-20211108_082107-2idnuxcc/files/config.yaml CHANGED
@@ -23,6 +23,12 @@ _wandb:
23
  - 3
24
  - 11
25
  - 12
 
 
 
 
 
 
26
  3:
27
  - 13
28
  4: 3.8.10
 
23
  - 3
24
  - 11
25
  - 12
26
+ 2:
27
+ - 1
28
+ - 2
29
+ - 3
30
+ - 11
31
+ - 12
32
  3:
33
  - 13
34
  4: 3.8.10
wandb/run-20211108_082107-2idnuxcc/files/output.log CHANGED
@@ -12689,3 +12689,342 @@ tcmalloc: large alloc 2886115328 bytes == 0x4e0378000 @ 0x7f148c0d4680 0x7f148c
12689
  [15:45:06] - INFO - absl - Saved checkpoint at ./checkpoint_100000
12690
  [15:45:06] - INFO - absl - Removing checkpoint at ./checkpoint_75000
12691
  Adding files tracked by Git LFS: ['checkpoint_100000']. This may take a bit of time if the files are large.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
12689
  [15:45:06] - INFO - absl - Saved checkpoint at ./checkpoint_100000
12690
  [15:45:06] - INFO - absl - Removing checkpoint at ./checkpoint_75000
12691
  Adding files tracked by Git LFS: ['checkpoint_100000']. This may take a bit of time if the files are large.
12692
+ [15:45:37] - WARNING - huggingface_hub.repository - Adding files tracked by Git LFS: ['checkpoint_100000']. This may take a bit of time if the files are large.
12693
+
12694
+
12695
+
12696
+
12697
+
12698
+
12699
+
12700
+
12701
+
12702
+
12703
+
12704
+
12705
+
12706
+
12707
+
12708
+
12709
+
12710
+
12711
+
12712
+
12713
+
12714
+
12715
+
12716
+
12717
+
12718
+
12719
+
12720
+
12721
+
12722
+
12723
+
12724
+
12725
+
12726
+
12727
+
12728
+
12729
+
12730
+
12731
+
12732
+
12733
+
12734
+
12735
+
12736
+
12737
+
12738
+
12739
+
12740
+
12741
+
12742
+
12743
+
12744
+
12745
+
12746
+
12747
+
12748
+
12749
+
12750
+
12751
+
12752
+
12753
+
12754
+
12755
+
12756
+
12757
+
12758
+
12759
+
12760
+
12761
+
12762
+
12763
+
12764
+
12765
+
12766
+
12767
+
12768
+
12769
+
12770
+
12771
+
12772
+
12773
+
12774
+
12775
+
12776
+
12777
+
12778
+
12779
+
12780
+
12781
+
12782
+
12783
+
12784
+
12785
+
12786
+
12787
+
12788
+
12789
+
12790
+
12791
+
12792
+
12793
+
12794
+
12795
+
12796
+
12797
+
12798
+
12799
+
12800
+
12801
+
12802
+
12803
+
12804
+
12805
+
12806
+
12807
+
12808
+
12809
+
12810
+
12811
+
12812
+
12813
+
12814
+
12815
+
12816
+
12817
+
12818
+
12819
+
12820
+
12821
+
12822
+
12823
+
12824
+
12825
+
12826
+
12827
+
12828
+
12829
+
12830
+
12831
+
12832
+
12833
+
12834
+
12835
+
12836
+
12837
+
12838
+
12839
+
12840
+
12841
+
12842
+
12843
+
12844
+
12845
+
12846
+
12847
+
12848
+
12849
+
12850
+
12851
+
12852
+
12853
+
12854
+
12855
+
12856
+
12857
+
12858
+
12859
+
12860
+
12861
+
12862
+
12863
+
12864
+
12865
+
12866
+
12867
+
12868
+
12869
+
12870
+
12871
+
12872
+
12873
+
12874
+
12875
+
12876
+
12877
+
12878
+
12879
+
12880
+
12881
+
12882
+
12883
+
12884
+
12885
+
12886
+
12887
+
12888
+
12889
+
12890
+
12891
+
12892
+
12893
+
12894
+
12895
+
12896
+
12897
+
12898
+
12899
+
12900
+
12901
+
12902
+
12903
+
12904
+
12905
+
12906
+
12907
+
12908
+
12909
+
12910
+
12911
+
12912
+
12913
+
12914
+
12915
+
12916
+
12917
+
12918
+
12919
+
12920
+
12921
+
12922
+
12923
+
12924
+
12925
+
12926
+ Training...: 83%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 50999/61250 [7:29:40<1:22:52, 2.06it/s]
12927
+
12928
+
12929
+
12930
+
12931
+
12932
+
12933
+
12934
+
12935
+
12936
+
12937
+
12938
+
12939
+
12940
+
12941
+
12942
+
12943
+
12944
+
12945
+
12946
+
12947
+
12948
+
12949
+
12950
+
12951
+
12952
+
12953
+
12954
+
12955
+
12956
+
12957
+
12958
+
12959
+
12960
+
12961
+
12962
+
12963
+
12964
+
12965
+
12966
+
12967
+
12968
+
12969
+
12970
+
12971
+
12972
+
12973
+
12974
+
12975
+
12976
+
12977
+
12978
+
12979
+
12980
+
12981
+
12982
+
12983
+
12984
+
12985
+
12986
+
12987
+
12988
+
12989
+
12990
+
12991
+
12992
+
12993
+
12994
+
12995
+
12996
+
12997
+
12998
+
12999
+
13000
+
13001
+
13002
+
13003
+
13004
+
13005
+
13006
+
13007
+
13008
+
13009
+
13010
+
13011
+
13012
+
13013
+
13014
+
13015
+
13016
+
13017
+
13018
+
13019
+
13020
+
13021
+
13022
+
13023
+
13024
+
13025
+ Step... (100000 | Loss: 1.2511297464370728, Acc: 0.748236894607544): 0%| | 0/3 [7:34:20<?, ?it/s]
13026
+ Traceback (most recent call last):
13027
+ File "run_muril_bb_flax_mlm.py", line 695, in <module>
13028
+ state, train_metric, dropout_rngs = p_train_step(state, model_inputs, dropout_rngs)
13029
+ File "<string>", line 1, in __new__
13030
+ KeyboardInterrupt
wandb/run-20211108_082107-2idnuxcc/files/wandb-summary.json CHANGED
@@ -1 +1 @@
1
- {"training_step": 100000, "learning_rate": 2.2789117792854086e-05, "train_loss": 1.3630295991897583, "_runtime": 26628, "_timestamp": 1636386295, "_step": 59, "eval_step": 100000, "eval_accuracy": 0.748236894607544, "eval_loss": 1.2511297464370728}
 
1
+ {"training_step": 101000, "learning_rate": 2.2517007892020047e-05, "train_loss": 1.055612325668335, "_runtime": 27185, "_timestamp": 1636386852, "_step": 60, "eval_step": 100000, "eval_accuracy": 0.748236894607544, "eval_loss": 1.2511297464370728, "_wandb": {"runtime": 27383}}
wandb/run-20211108_082107-2idnuxcc/logs/debug-internal.log CHANGED
@@ -17336,3 +17336,566 @@
17336
  2021-11-08 15:45:41,901 DEBUG SenderThread:5242 [sender.py:send():238] send: stats
17337
  2021-11-08 15:45:43,920 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17338
  2021-11-08 15:45:43,920 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
17336
  2021-11-08 15:45:41,901 DEBUG SenderThread:5242 [sender.py:send():238] send: stats
17337
  2021-11-08 15:45:43,920 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17338
  2021-11-08 15:45:43,920 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17339
+ 2021-11-08 15:45:59,059 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17340
+ 2021-11-08 15:45:59,059 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17341
+ 2021-11-08 15:46:07,598 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17342
+ 2021-11-08 15:46:09,599 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17343
+ 2021-11-08 15:46:11,600 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17344
+ 2021-11-08 15:46:11,979 DEBUG SenderThread:5242 [sender.py:send():238] send: stats
17345
+ 2021-11-08 15:46:13,600 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17346
+ 2021-11-08 15:46:14,203 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17347
+ 2021-11-08 15:46:14,203 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17348
+ 2021-11-08 15:46:15,601 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17349
+ 2021-11-08 15:46:17,602 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17350
+ 2021-11-08 15:46:19,603 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17351
+ 2021-11-08 15:46:21,604 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17352
+ 2021-11-08 15:46:23,605 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17353
+ 2021-11-08 15:46:25,606 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17354
+ 2021-11-08 15:46:27,607 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17355
+ 2021-11-08 15:46:29,358 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17356
+ 2021-11-08 15:46:29,358 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17357
+ 2021-11-08 15:46:29,608 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17358
+ 2021-11-08 15:46:32,609 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17359
+ 2021-11-08 15:46:34,610 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17360
+ 2021-11-08 15:46:36,611 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17361
+ 2021-11-08 15:46:38,612 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17362
+ 2021-11-08 15:46:40,613 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17363
+ 2021-11-08 15:46:42,054 DEBUG SenderThread:5242 [sender.py:send():238] send: stats
17364
+ 2021-11-08 15:46:42,614 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17365
+ 2021-11-08 15:46:44,507 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17366
+ 2021-11-08 15:46:44,508 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17367
+ 2021-11-08 15:46:44,615 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17368
+ 2021-11-08 15:46:46,616 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17369
+ 2021-11-08 15:46:48,617 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17370
+ 2021-11-08 15:46:50,618 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17371
+ 2021-11-08 15:46:52,618 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17372
+ 2021-11-08 15:46:54,619 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17373
+ 2021-11-08 15:46:56,620 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17374
+ 2021-11-08 15:46:58,621 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17375
+ 2021-11-08 15:46:59,641 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17376
+ 2021-11-08 15:46:59,641 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17377
+ 2021-11-08 15:47:00,622 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17378
+ 2021-11-08 15:47:02,623 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17379
+ 2021-11-08 15:47:04,624 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17380
+ 2021-11-08 15:47:06,625 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17381
+ 2021-11-08 15:47:08,626 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17382
+ 2021-11-08 15:47:10,626 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17383
+ 2021-11-08 15:47:12,128 DEBUG SenderThread:5242 [sender.py:send():238] send: stats
17384
+ 2021-11-08 15:47:12,627 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17385
+ 2021-11-08 15:47:14,628 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17386
+ 2021-11-08 15:47:14,786 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17387
+ 2021-11-08 15:47:14,786 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17388
+ 2021-11-08 15:47:16,629 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17389
+ 2021-11-08 15:47:18,630 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17390
+ 2021-11-08 15:47:20,631 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17391
+ 2021-11-08 15:47:22,632 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17392
+ 2021-11-08 15:47:24,633 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17393
+ 2021-11-08 15:47:26,634 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17394
+ 2021-11-08 15:47:28,635 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17395
+ 2021-11-08 15:47:29,991 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17396
+ 2021-11-08 15:47:29,991 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17397
+ 2021-11-08 15:47:30,635 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17398
+ 2021-11-08 15:47:32,636 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17399
+ 2021-11-08 15:47:34,637 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17400
+ 2021-11-08 15:47:36,638 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17401
+ 2021-11-08 15:47:38,639 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17402
+ 2021-11-08 15:47:40,640 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17403
+ 2021-11-08 15:47:42,202 DEBUG SenderThread:5242 [sender.py:send():238] send: stats
17404
+ 2021-11-08 15:47:42,641 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17405
+ 2021-11-08 15:47:44,642 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17406
+ 2021-11-08 15:47:45,132 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17407
+ 2021-11-08 15:47:45,132 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17408
+ 2021-11-08 15:47:46,642 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17409
+ 2021-11-08 15:47:48,643 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17410
+ 2021-11-08 15:47:50,644 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17411
+ 2021-11-08 15:47:52,645 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17412
+ 2021-11-08 15:47:54,646 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17413
+ 2021-11-08 15:47:56,647 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17414
+ 2021-11-08 15:47:58,648 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17415
+ 2021-11-08 15:48:00,267 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17416
+ 2021-11-08 15:48:00,267 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17417
+ 2021-11-08 15:48:00,649 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17418
+ 2021-11-08 15:48:02,650 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17419
+ 2021-11-08 15:48:04,651 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17420
+ 2021-11-08 15:48:06,652 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17421
+ 2021-11-08 15:48:08,653 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17422
+ 2021-11-08 15:48:10,654 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17423
+ 2021-11-08 15:48:12,277 DEBUG SenderThread:5242 [sender.py:send():238] send: stats
17424
+ 2021-11-08 15:48:12,655 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17425
+ 2021-11-08 15:48:14,655 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17426
+ 2021-11-08 15:48:15,400 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17427
+ 2021-11-08 15:48:15,400 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17428
+ 2021-11-08 15:48:16,656 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17429
+ 2021-11-08 15:48:18,657 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17430
+ 2021-11-08 15:48:20,658 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17431
+ 2021-11-08 15:48:22,659 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17432
+ 2021-11-08 15:48:24,660 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17433
+ 2021-11-08 15:48:26,661 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17434
+ 2021-11-08 15:48:28,662 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17435
+ 2021-11-08 15:48:30,535 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17436
+ 2021-11-08 15:48:30,535 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17437
+ 2021-11-08 15:48:30,663 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17438
+ 2021-11-08 15:48:32,664 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17439
+ 2021-11-08 15:48:34,664 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17440
+ 2021-11-08 15:48:36,665 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17441
+ 2021-11-08 15:48:38,666 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17442
+ 2021-11-08 15:48:40,667 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17443
+ 2021-11-08 15:48:42,352 DEBUG SenderThread:5242 [sender.py:send():238] send: stats
17444
+ 2021-11-08 15:48:42,668 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17445
+ 2021-11-08 15:48:44,669 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17446
+ 2021-11-08 15:48:45,670 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17447
+ 2021-11-08 15:48:45,670 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17448
+ 2021-11-08 15:48:46,670 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17449
+ 2021-11-08 15:48:48,671 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17450
+ 2021-11-08 15:48:50,672 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17451
+ 2021-11-08 15:48:52,672 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17452
+ 2021-11-08 15:48:54,673 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17453
+ 2021-11-08 15:48:56,674 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17454
+ 2021-11-08 15:48:58,675 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17455
+ 2021-11-08 15:49:00,676 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17456
+ 2021-11-08 15:49:00,803 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17457
+ 2021-11-08 15:49:00,803 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17458
+ 2021-11-08 15:49:02,677 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17459
+ 2021-11-08 15:49:04,678 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17460
+ 2021-11-08 15:49:06,679 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17461
+ 2021-11-08 15:49:08,679 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17462
+ 2021-11-08 15:49:10,680 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17463
+ 2021-11-08 15:49:12,425 DEBUG SenderThread:5242 [sender.py:send():238] send: stats
17464
+ 2021-11-08 15:49:12,681 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17465
+ 2021-11-08 15:49:14,682 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17466
+ 2021-11-08 15:49:15,939 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17467
+ 2021-11-08 15:49:15,939 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17468
+ 2021-11-08 15:49:16,683 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17469
+ 2021-11-08 15:49:18,683 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17470
+ 2021-11-08 15:49:20,684 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17471
+ 2021-11-08 15:49:22,685 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17472
+ 2021-11-08 15:49:24,686 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17473
+ 2021-11-08 15:49:26,687 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17474
+ 2021-11-08 15:49:28,688 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17475
+ 2021-11-08 15:49:30,689 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17476
+ 2021-11-08 15:49:31,072 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17477
+ 2021-11-08 15:49:31,072 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17478
+ 2021-11-08 15:49:32,690 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17479
+ 2021-11-08 15:49:34,691 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17480
+ 2021-11-08 15:49:36,691 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17481
+ 2021-11-08 15:49:38,692 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17482
+ 2021-11-08 15:49:40,693 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17483
+ 2021-11-08 15:49:42,500 DEBUG SenderThread:5242 [sender.py:send():238] send: stats
17484
+ 2021-11-08 15:49:42,694 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17485
+ 2021-11-08 15:49:44,695 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17486
+ 2021-11-08 15:49:46,221 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17487
+ 2021-11-08 15:49:46,221 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17488
+ 2021-11-08 15:49:46,696 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17489
+ 2021-11-08 15:49:48,697 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17490
+ 2021-11-08 15:49:50,698 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17491
+ 2021-11-08 15:49:52,699 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17492
+ 2021-11-08 15:49:54,700 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17493
+ 2021-11-08 15:49:56,700 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17494
+ 2021-11-08 15:49:58,701 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17495
+ 2021-11-08 15:50:00,702 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17496
+ 2021-11-08 15:50:01,353 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17497
+ 2021-11-08 15:50:01,354 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17498
+ 2021-11-08 15:50:02,703 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17499
+ 2021-11-08 15:50:04,704 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17500
+ 2021-11-08 15:50:06,704 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17501
+ 2021-11-08 15:50:08,706 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17502
+ 2021-11-08 15:50:10,707 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17503
+ 2021-11-08 15:50:12,576 DEBUG SenderThread:5242 [sender.py:send():238] send: stats
17504
+ 2021-11-08 15:50:12,707 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17505
+ 2021-11-08 15:50:14,708 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17506
+ 2021-11-08 15:50:16,493 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17507
+ 2021-11-08 15:50:16,493 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17508
+ 2021-11-08 15:50:16,709 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17509
+ 2021-11-08 15:50:18,710 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17510
+ 2021-11-08 15:50:20,710 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17511
+ 2021-11-08 15:50:22,711 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17512
+ 2021-11-08 15:50:24,712 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17513
+ 2021-11-08 15:50:26,713 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17514
+ 2021-11-08 15:50:28,714 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17515
+ 2021-11-08 15:50:30,714 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17516
+ 2021-11-08 15:50:31,643 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17517
+ 2021-11-08 15:50:31,644 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17518
+ 2021-11-08 15:50:32,715 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17519
+ 2021-11-08 15:50:34,715 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17520
+ 2021-11-08 15:50:36,716 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17521
+ 2021-11-08 15:50:38,717 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17522
+ 2021-11-08 15:50:40,718 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17523
+ 2021-11-08 15:50:42,652 DEBUG SenderThread:5242 [sender.py:send():238] send: stats
17524
+ 2021-11-08 15:50:42,719 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17525
+ 2021-11-08 15:50:45,720 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17526
+ 2021-11-08 15:50:46,779 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17527
+ 2021-11-08 15:50:46,779 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17528
+ 2021-11-08 15:50:47,720 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17529
+ 2021-11-08 15:50:49,721 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17530
+ 2021-11-08 15:50:51,722 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17531
+ 2021-11-08 15:50:53,723 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17532
+ 2021-11-08 15:50:55,724 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17533
+ 2021-11-08 15:50:57,725 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17534
+ 2021-11-08 15:50:59,726 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17535
+ 2021-11-08 15:51:01,726 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17536
+ 2021-11-08 15:51:01,948 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17537
+ 2021-11-08 15:51:01,948 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17538
+ 2021-11-08 15:51:03,727 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17539
+ 2021-11-08 15:51:05,728 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17540
+ 2021-11-08 15:51:07,729 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17541
+ 2021-11-08 15:51:09,730 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17542
+ 2021-11-08 15:51:11,731 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17543
+ 2021-11-08 15:51:12,729 DEBUG SenderThread:5242 [sender.py:send():238] send: stats
17544
+ 2021-11-08 15:51:13,732 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17545
+ 2021-11-08 15:51:15,733 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17546
+ 2021-11-08 15:51:17,080 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17547
+ 2021-11-08 15:51:17,080 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17548
+ 2021-11-08 15:51:17,734 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17549
+ 2021-11-08 15:51:19,735 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17550
+ 2021-11-08 15:51:21,736 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17551
+ 2021-11-08 15:51:23,737 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17552
+ 2021-11-08 15:51:25,738 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17553
+ 2021-11-08 15:51:27,738 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17554
+ 2021-11-08 15:51:29,739 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17555
+ 2021-11-08 15:51:31,740 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17556
+ 2021-11-08 15:51:32,212 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17557
+ 2021-11-08 15:51:32,212 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17558
+ 2021-11-08 15:51:33,741 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17559
+ 2021-11-08 15:51:35,742 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17560
+ 2021-11-08 15:51:37,743 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17561
+ 2021-11-08 15:51:39,744 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17562
+ 2021-11-08 15:51:41,745 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17563
+ 2021-11-08 15:51:42,805 DEBUG SenderThread:5242 [sender.py:send():238] send: stats
17564
+ 2021-11-08 15:51:43,746 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17565
+ 2021-11-08 15:51:45,747 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17566
+ 2021-11-08 15:51:47,347 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17567
+ 2021-11-08 15:51:47,348 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17568
+ 2021-11-08 15:51:47,748 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17569
+ 2021-11-08 15:51:49,749 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17570
+ 2021-11-08 15:51:51,749 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17571
+ 2021-11-08 15:51:53,750 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17572
+ 2021-11-08 15:51:55,751 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17573
+ 2021-11-08 15:51:57,752 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17574
+ 2021-11-08 15:51:59,753 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17575
+ 2021-11-08 15:52:01,754 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17576
+ 2021-11-08 15:52:02,485 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17577
+ 2021-11-08 15:52:02,485 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17578
+ 2021-11-08 15:52:03,755 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17579
+ 2021-11-08 15:52:05,756 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17580
+ 2021-11-08 15:52:07,757 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17581
+ 2021-11-08 15:52:09,757 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17582
+ 2021-11-08 15:52:11,758 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17583
+ 2021-11-08 15:52:12,880 DEBUG SenderThread:5242 [sender.py:send():238] send: stats
17584
+ 2021-11-08 15:52:13,759 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17585
+ 2021-11-08 15:52:15,760 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17586
+ 2021-11-08 15:52:17,641 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17587
+ 2021-11-08 15:52:17,642 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17588
+ 2021-11-08 15:52:17,761 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17589
+ 2021-11-08 15:52:19,762 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17590
+ 2021-11-08 15:52:21,763 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17591
+ 2021-11-08 15:52:23,763 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17592
+ 2021-11-08 15:52:25,764 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17593
+ 2021-11-08 15:52:27,765 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17594
+ 2021-11-08 15:52:29,766 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17595
+ 2021-11-08 15:52:31,767 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17596
+ 2021-11-08 15:52:32,774 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17597
+ 2021-11-08 15:52:32,774 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17598
+ 2021-11-08 15:52:33,768 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17599
+ 2021-11-08 15:52:35,769 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17600
+ 2021-11-08 15:52:37,770 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17601
+ 2021-11-08 15:52:39,771 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17602
+ 2021-11-08 15:52:41,772 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17603
+ 2021-11-08 15:52:42,954 DEBUG SenderThread:5242 [sender.py:send():238] send: stats
17604
+ 2021-11-08 15:52:43,773 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17605
+ 2021-11-08 15:52:45,773 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17606
+ 2021-11-08 15:52:47,774 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17607
+ 2021-11-08 15:52:47,911 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17608
+ 2021-11-08 15:52:47,911 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17609
+ 2021-11-08 15:52:49,775 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17610
+ 2021-11-08 15:52:51,776 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17611
+ 2021-11-08 15:52:53,776 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17612
+ 2021-11-08 15:52:55,777 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17613
+ 2021-11-08 15:52:57,778 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17614
+ 2021-11-08 15:52:59,778 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17615
+ 2021-11-08 15:53:01,779 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17616
+ 2021-11-08 15:53:03,046 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17617
+ 2021-11-08 15:53:03,046 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17618
+ 2021-11-08 15:53:03,780 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17619
+ 2021-11-08 15:53:05,781 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17620
+ 2021-11-08 15:53:07,781 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17621
+ 2021-11-08 15:53:09,782 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17622
+ 2021-11-08 15:53:11,783 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17623
+ 2021-11-08 15:53:13,027 DEBUG SenderThread:5242 [sender.py:send():238] send: stats
17624
+ 2021-11-08 15:53:13,783 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17625
+ 2021-11-08 15:53:15,784 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17626
+ 2021-11-08 15:53:17,785 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17627
+ 2021-11-08 15:53:18,181 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17628
+ 2021-11-08 15:53:18,182 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17629
+ 2021-11-08 15:53:19,786 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17630
+ 2021-11-08 15:53:21,787 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17631
+ 2021-11-08 15:53:23,787 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17632
+ 2021-11-08 15:53:25,788 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17633
+ 2021-11-08 15:53:27,789 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17634
+ 2021-11-08 15:53:29,790 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17635
+ 2021-11-08 15:53:31,790 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17636
+ 2021-11-08 15:53:33,317 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17637
+ 2021-11-08 15:53:33,317 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17638
+ 2021-11-08 15:53:33,791 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17639
+ 2021-11-08 15:53:35,792 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17640
+ 2021-11-08 15:53:37,793 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17641
+ 2021-11-08 15:53:39,794 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17642
+ 2021-11-08 15:53:41,795 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17643
+ 2021-11-08 15:53:43,103 DEBUG SenderThread:5242 [sender.py:send():238] send: stats
17644
+ 2021-11-08 15:53:43,795 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17645
+ 2021-11-08 15:53:45,796 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17646
+ 2021-11-08 15:53:47,797 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17647
+ 2021-11-08 15:53:48,450 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17648
+ 2021-11-08 15:53:48,451 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17649
+ 2021-11-08 15:53:49,798 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17650
+ 2021-11-08 15:53:51,799 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17651
+ 2021-11-08 15:53:53,799 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17652
+ 2021-11-08 15:53:55,800 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17653
+ 2021-11-08 15:54:03,585 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17654
+ 2021-11-08 15:54:03,585 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17655
+ 2021-11-08 15:54:12,164 DEBUG SenderThread:5242 [sender.py:send():238] send: history
17656
+ 2021-11-08 15:54:12,164 DEBUG SenderThread:5242 [sender.py:send():238] send: summary
17657
+ 2021-11-08 15:54:12,165 INFO SenderThread:5242 [sender.py:_save_file():960] saving file wandb-summary.json with policy end
17658
+ 2021-11-08 15:54:12,806 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/wandb-summary.json
17659
+ 2021-11-08 15:54:12,806 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17660
+ 2021-11-08 15:54:13,178 DEBUG SenderThread:5242 [sender.py:send():238] send: stats
17661
+ 2021-11-08 15:54:13,806 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17662
+ 2021-11-08 15:54:15,807 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17663
+ 2021-11-08 15:54:17,807 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17664
+ 2021-11-08 15:54:18,724 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17665
+ 2021-11-08 15:54:18,724 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17666
+ 2021-11-08 15:54:19,808 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17667
+ 2021-11-08 15:54:21,809 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17668
+ 2021-11-08 15:54:23,810 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17669
+ 2021-11-08 15:54:25,810 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17670
+ 2021-11-08 15:54:27,811 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17671
+ 2021-11-08 15:54:29,812 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17672
+ 2021-11-08 15:54:31,813 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17673
+ 2021-11-08 15:54:33,813 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17674
+ 2021-11-08 15:54:33,857 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17675
+ 2021-11-08 15:54:33,858 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17676
+ 2021-11-08 15:54:35,814 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17677
+ 2021-11-08 15:54:37,815 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17678
+ 2021-11-08 15:54:39,816 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17679
+ 2021-11-08 15:54:41,817 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17680
+ 2021-11-08 15:54:43,254 DEBUG SenderThread:5242 [sender.py:send():238] send: stats
17681
+ 2021-11-08 15:54:43,817 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17682
+ 2021-11-08 15:54:45,818 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17683
+ 2021-11-08 15:54:47,819 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17684
+ 2021-11-08 15:54:49,002 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17685
+ 2021-11-08 15:54:49,003 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17686
+ 2021-11-08 15:54:49,819 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17687
+ 2021-11-08 15:54:51,820 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17688
+ 2021-11-08 15:54:53,821 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17689
+ 2021-11-08 15:54:55,821 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17690
+ 2021-11-08 15:54:57,822 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17691
+ 2021-11-08 15:54:59,823 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17692
+ 2021-11-08 15:55:01,824 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17693
+ 2021-11-08 15:55:03,824 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17694
+ 2021-11-08 15:55:04,140 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17695
+ 2021-11-08 15:55:04,140 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17696
+ 2021-11-08 15:55:05,825 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17697
+ 2021-11-08 15:55:07,826 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17698
+ 2021-11-08 15:55:09,826 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17699
+ 2021-11-08 15:55:11,827 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17700
+ 2021-11-08 15:55:13,328 DEBUG SenderThread:5242 [sender.py:send():238] send: stats
17701
+ 2021-11-08 15:55:13,828 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17702
+ 2021-11-08 15:55:15,829 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17703
+ 2021-11-08 15:55:17,829 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17704
+ 2021-11-08 15:55:19,291 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17705
+ 2021-11-08 15:55:19,291 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17706
+ 2021-11-08 15:55:20,831 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17707
+ 2021-11-08 15:55:22,831 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17708
+ 2021-11-08 15:55:24,832 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17709
+ 2021-11-08 15:55:26,833 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17710
+ 2021-11-08 15:55:28,834 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17711
+ 2021-11-08 15:55:30,835 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17712
+ 2021-11-08 15:55:32,835 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17713
+ 2021-11-08 15:55:34,431 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17714
+ 2021-11-08 15:55:34,432 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17715
+ 2021-11-08 15:55:34,836 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17716
+ 2021-11-08 15:55:36,837 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17717
+ 2021-11-08 15:55:38,837 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17718
+ 2021-11-08 15:55:40,838 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17719
+ 2021-11-08 15:55:42,839 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17720
+ 2021-11-08 15:55:43,398 DEBUG SenderThread:5242 [sender.py:send():238] send: stats
17721
+ 2021-11-08 15:55:44,840 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17722
+ 2021-11-08 15:55:46,841 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17723
+ 2021-11-08 15:55:48,841 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17724
+ 2021-11-08 15:55:49,566 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17725
+ 2021-11-08 15:55:49,566 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17726
+ 2021-11-08 15:55:50,842 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17727
+ 2021-11-08 15:55:52,843 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17728
+ 2021-11-08 15:55:54,844 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17729
+ 2021-11-08 15:55:56,844 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17730
+ 2021-11-08 15:55:58,845 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17731
+ 2021-11-08 15:56:00,846 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17732
+ 2021-11-08 15:56:02,847 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17733
+ 2021-11-08 15:56:04,700 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17734
+ 2021-11-08 15:56:04,700 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17735
+ 2021-11-08 15:56:04,848 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17736
+ 2021-11-08 15:56:06,849 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17737
+ 2021-11-08 15:56:08,849 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17738
+ 2021-11-08 15:56:10,850 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17739
+ 2021-11-08 15:56:12,851 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17740
+ 2021-11-08 15:56:13,473 DEBUG SenderThread:5242 [sender.py:send():238] send: stats
17741
+ 2021-11-08 15:56:14,852 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17742
+ 2021-11-08 15:56:16,852 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17743
+ 2021-11-08 15:56:18,853 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17744
+ 2021-11-08 15:56:19,834 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17745
+ 2021-11-08 15:56:19,834 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17746
+ 2021-11-08 15:56:20,854 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17747
+ 2021-11-08 15:56:22,854 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17748
+ 2021-11-08 15:56:24,855 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17749
+ 2021-11-08 15:56:26,856 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17750
+ 2021-11-08 15:56:28,857 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17751
+ 2021-11-08 15:56:30,858 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17752
+ 2021-11-08 15:56:32,859 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17753
+ 2021-11-08 15:56:34,859 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17754
+ 2021-11-08 15:56:34,968 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17755
+ 2021-11-08 15:56:34,969 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17756
+ 2021-11-08 15:56:36,860 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17757
+ 2021-11-08 15:56:38,861 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17758
+ 2021-11-08 15:56:40,862 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17759
+ 2021-11-08 15:56:42,862 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17760
+ 2021-11-08 15:56:43,547 DEBUG SenderThread:5242 [sender.py:send():238] send: stats
17761
+ 2021-11-08 15:56:44,863 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17762
+ 2021-11-08 15:56:46,864 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17763
+ 2021-11-08 15:56:48,864 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17764
+ 2021-11-08 15:56:50,111 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17765
+ 2021-11-08 15:56:50,111 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17766
+ 2021-11-08 15:56:50,865 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17767
+ 2021-11-08 15:56:52,866 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17768
+ 2021-11-08 15:56:54,867 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17769
+ 2021-11-08 15:56:56,867 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17770
+ 2021-11-08 15:56:58,868 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17771
+ 2021-11-08 15:57:00,869 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17772
+ 2021-11-08 15:57:02,870 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17773
+ 2021-11-08 15:57:04,871 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17774
+ 2021-11-08 15:57:05,246 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17775
+ 2021-11-08 15:57:05,247 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17776
+ 2021-11-08 15:57:06,871 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17777
+ 2021-11-08 15:57:08,872 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17778
+ 2021-11-08 15:57:10,873 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17779
+ 2021-11-08 15:57:12,873 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17780
+ 2021-11-08 15:57:13,622 DEBUG SenderThread:5242 [sender.py:send():238] send: stats
17781
+ 2021-11-08 15:57:14,874 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17782
+ 2021-11-08 15:57:16,875 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17783
+ 2021-11-08 15:57:18,876 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17784
+ 2021-11-08 15:57:20,390 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: stop_status
17785
+ 2021-11-08 15:57:20,390 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: stop_status
17786
+ 2021-11-08 15:57:20,877 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17787
+ 2021-11-08 15:57:22,877 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17788
+ 2021-11-08 15:57:24,878 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17789
+ 2021-11-08 15:57:26,879 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17790
+ 2021-11-08 15:57:28,879 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17791
+ 2021-11-08 15:57:29,070 WARNING MainThread:5242 [internal.py:wandb_internal():146] Internal process interrupt: 1
17792
+ 2021-11-08 15:57:30,880 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17793
+ 2021-11-08 15:57:31,724 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: poll_exit
17794
+ 2021-11-08 15:57:31,724 DEBUG SenderThread:5242 [sender.py:send():238] send: telemetry
17795
+ 2021-11-08 15:57:31,724 DEBUG SenderThread:5242 [sender.py:send():238] send: exit
17796
+ 2021-11-08 15:57:31,724 INFO SenderThread:5242 [sender.py:send_exit():370] handling exit code: 255
17797
+ 2021-11-08 15:57:31,725 INFO SenderThread:5242 [sender.py:send_exit():372] handling runtime: 27383
17798
+ 2021-11-08 15:57:31,725 INFO SenderThread:5242 [sender.py:_save_file():960] saving file wandb-summary.json with policy end
17799
+ 2021-11-08 15:57:31,726 INFO SenderThread:5242 [sender.py:send_exit():382] send defer
17800
+ 2021-11-08 15:57:31,726 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: poll_exit
17801
+ 2021-11-08 15:57:31,726 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: defer
17802
+ 2021-11-08 15:57:31,726 INFO HandlerThread:5242 [handler.py:handle_request_defer():147] handle defer: 0
17803
+ 2021-11-08 15:57:31,726 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: defer
17804
+ 2021-11-08 15:57:31,727 INFO SenderThread:5242 [sender.py:send_request_defer():391] handle sender defer: 0
17805
+ 2021-11-08 15:57:31,727 INFO SenderThread:5242 [sender.py:transition_state():395] send defer: 1
17806
+ 2021-11-08 15:57:31,727 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: defer
17807
+ 2021-11-08 15:57:31,727 INFO HandlerThread:5242 [handler.py:handle_request_defer():147] handle defer: 1
17808
+ 2021-11-08 15:57:31,766 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: defer
17809
+ 2021-11-08 15:57:31,767 INFO SenderThread:5242 [sender.py:send_request_defer():391] handle sender defer: 1
17810
+ 2021-11-08 15:57:31,767 INFO SenderThread:5242 [sender.py:transition_state():395] send defer: 2
17811
+ 2021-11-08 15:57:31,767 DEBUG SenderThread:5242 [sender.py:send():238] send: stats
17812
+ 2021-11-08 15:57:31,767 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: defer
17813
+ 2021-11-08 15:57:31,767 INFO HandlerThread:5242 [handler.py:handle_request_defer():147] handle defer: 2
17814
+ 2021-11-08 15:57:31,767 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: defer
17815
+ 2021-11-08 15:57:31,767 INFO SenderThread:5242 [sender.py:send_request_defer():391] handle sender defer: 2
17816
+ 2021-11-08 15:57:31,767 INFO SenderThread:5242 [sender.py:transition_state():395] send defer: 3
17817
+ 2021-11-08 15:57:31,768 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: defer
17818
+ 2021-11-08 15:57:31,768 INFO HandlerThread:5242 [handler.py:handle_request_defer():147] handle defer: 3
17819
+ 2021-11-08 15:57:31,768 DEBUG SenderThread:5242 [sender.py:send():238] send: summary
17820
+ 2021-11-08 15:57:31,768 INFO SenderThread:5242 [sender.py:_save_file():960] saving file wandb-summary.json with policy end
17821
+ 2021-11-08 15:57:31,768 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: defer
17822
+ 2021-11-08 15:57:31,768 INFO SenderThread:5242 [sender.py:send_request_defer():391] handle sender defer: 3
17823
+ 2021-11-08 15:57:31,768 INFO SenderThread:5242 [sender.py:transition_state():395] send defer: 4
17824
+ 2021-11-08 15:57:31,769 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: defer
17825
+ 2021-11-08 15:57:31,769 INFO HandlerThread:5242 [handler.py:handle_request_defer():147] handle defer: 4
17826
+ 2021-11-08 15:57:31,769 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: defer
17827
+ 2021-11-08 15:57:31,769 INFO SenderThread:5242 [sender.py:send_request_defer():391] handle sender defer: 4
17828
+ 2021-11-08 15:57:31,828 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: poll_exit
17829
+ 2021-11-08 15:57:31,881 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/wandb-summary.json
17830
+ 2021-11-08 15:57:31,881 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17831
+ 2021-11-08 15:57:31,984 INFO SenderThread:5242 [sender.py:transition_state():395] send defer: 5
17832
+ 2021-11-08 15:57:31,985 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: poll_exit
17833
+ 2021-11-08 15:57:31,985 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: defer
17834
+ 2021-11-08 15:57:31,985 INFO HandlerThread:5242 [handler.py:handle_request_defer():147] handle defer: 5
17835
+ 2021-11-08 15:57:31,985 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: defer
17836
+ 2021-11-08 15:57:31,986 INFO SenderThread:5242 [sender.py:send_request_defer():391] handle sender defer: 5
17837
+ 2021-11-08 15:57:31,986 INFO SenderThread:5242 [dir_watcher.py:finish():283] shutting down directory watcher
17838
+ 2021-11-08 15:57:32,086 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: poll_exit
17839
+ 2021-11-08 15:57:32,881 INFO Thread-8 :5242 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/config.yaml
17840
+ 2021-11-08 15:57:32,882 INFO SenderThread:5242 [dir_watcher.py:finish():313] scan: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files
17841
+ 2021-11-08 15:57:32,882 INFO SenderThread:5242 [dir_watcher.py:finish():327] scan save: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/requirements.txt requirements.txt
17842
+ 2021-11-08 15:57:32,882 INFO SenderThread:5242 [dir_watcher.py:finish():327] scan save: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log output.log
17843
+ 2021-11-08 15:57:32,883 INFO SenderThread:5242 [dir_watcher.py:finish():327] scan save: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/wandb-metadata.json wandb-metadata.json
17844
+ 2021-11-08 15:57:32,883 INFO SenderThread:5242 [dir_watcher.py:finish():327] scan save: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/config.yaml config.yaml
17845
+ 2021-11-08 15:57:32,883 INFO SenderThread:5242 [dir_watcher.py:finish():327] scan save: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/wandb-summary.json wandb-summary.json
17846
+ 2021-11-08 15:57:32,886 INFO SenderThread:5242 [sender.py:transition_state():395] send defer: 6
17847
+ 2021-11-08 15:57:32,886 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: poll_exit
17848
+ 2021-11-08 15:57:32,889 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: defer
17849
+ 2021-11-08 15:57:32,890 INFO HandlerThread:5242 [handler.py:handle_request_defer():147] handle defer: 6
17850
+ 2021-11-08 15:57:32,890 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: defer
17851
+ 2021-11-08 15:57:32,890 INFO SenderThread:5242 [sender.py:send_request_defer():391] handle sender defer: 6
17852
+ 2021-11-08 15:57:32,890 INFO SenderThread:5242 [file_pusher.py:finish():177] shutting down file pusher
17853
+ 2021-11-08 15:57:32,991 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: poll_exit
17854
+ 2021-11-08 15:57:32,992 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: poll_exit
17855
+ 2021-11-08 15:57:33,093 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: poll_exit
17856
+ 2021-11-08 15:57:33,093 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: poll_exit
17857
+ 2021-11-08 15:57:33,195 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: poll_exit
17858
+ 2021-11-08 15:57:33,196 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: poll_exit
17859
+ 2021-11-08 15:57:33,297 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: poll_exit
17860
+ 2021-11-08 15:57:33,298 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: poll_exit
17861
+ 2021-11-08 15:57:33,350 INFO Thread-14 :5242 [upload_job.py:push():137] Uploaded file /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/config.yaml
17862
+ 2021-11-08 15:57:33,388 INFO Thread-13 :5242 [upload_job.py:push():137] Uploaded file /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/output.log
17863
+ 2021-11-08 15:57:33,399 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: poll_exit
17864
+ 2021-11-08 15:57:33,399 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: poll_exit
17865
+ 2021-11-08 15:57:33,462 INFO Thread-15 :5242 [upload_job.py:push():137] Uploaded file /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/wandb-summary.json
17866
+ 2021-11-08 15:57:33,481 INFO Thread-12 :5242 [upload_job.py:push():137] Uploaded file /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/files/requirements.txt
17867
+ 2021-11-08 15:57:33,501 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: poll_exit
17868
+ 2021-11-08 15:57:33,501 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: poll_exit
17869
+ 2021-11-08 15:57:33,603 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: poll_exit
17870
+ 2021-11-08 15:57:33,603 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: poll_exit
17871
+ 2021-11-08 15:57:33,682 INFO Thread-7 :5242 [sender.py:transition_state():395] send defer: 7
17872
+ 2021-11-08 15:57:33,682 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: defer
17873
+ 2021-11-08 15:57:33,682 INFO HandlerThread:5242 [handler.py:handle_request_defer():147] handle defer: 7
17874
+ 2021-11-08 15:57:33,682 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: defer
17875
+ 2021-11-08 15:57:33,682 INFO SenderThread:5242 [sender.py:send_request_defer():391] handle sender defer: 7
17876
+ 2021-11-08 15:57:33,704 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: poll_exit
17877
+ 2021-11-08 15:57:33,967 INFO SenderThread:5242 [sender.py:transition_state():395] send defer: 8
17878
+ 2021-11-08 15:57:33,968 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: poll_exit
17879
+ 2021-11-08 15:57:33,968 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: defer
17880
+ 2021-11-08 15:57:33,968 INFO HandlerThread:5242 [handler.py:handle_request_defer():147] handle defer: 8
17881
+ 2021-11-08 15:57:33,968 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: defer
17882
+ 2021-11-08 15:57:33,968 INFO SenderThread:5242 [sender.py:send_request_defer():391] handle sender defer: 8
17883
+ 2021-11-08 15:57:33,969 INFO SenderThread:5242 [sender.py:transition_state():395] send defer: 9
17884
+ 2021-11-08 15:57:33,969 DEBUG SenderThread:5242 [sender.py:send():238] send: final
17885
+ 2021-11-08 15:57:33,969 DEBUG SenderThread:5242 [sender.py:send():238] send: footer
17886
+ 2021-11-08 15:57:33,969 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: defer
17887
+ 2021-11-08 15:57:33,969 INFO HandlerThread:5242 [handler.py:handle_request_defer():147] handle defer: 9
17888
+ 2021-11-08 15:57:33,969 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: defer
17889
+ 2021-11-08 15:57:33,969 INFO SenderThread:5242 [sender.py:send_request_defer():391] handle sender defer: 9
17890
+ 2021-11-08 15:57:34,069 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: poll_exit
17891
+ 2021-11-08 15:57:34,070 DEBUG SenderThread:5242 [sender.py:send_request():252] send_request: poll_exit
17892
+ 2021-11-08 15:57:34,070 INFO SenderThread:5242 [file_pusher.py:join():182] waiting for file pusher
17893
+ 2021-11-08 15:57:34,323 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: get_summary
17894
+ 2021-11-08 15:57:34,324 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: sampled_history
17895
+ 2021-11-08 15:57:34,326 DEBUG HandlerThread:5242 [handler.py:handle_request():130] handle_request: shutdown
17896
+ 2021-11-08 15:57:34,326 INFO HandlerThread:5242 [handler.py:finish():731] shutting down handler
17897
+ 2021-11-08 15:57:34,970 INFO WriterThread:5242 [datastore.py:close():281] close: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_082107-2idnuxcc/run-2idnuxcc.wandb
17898
+ 2021-11-08 15:57:35,322 INFO SenderThread:5242 [sender.py:finish():1091] shutting down sender
17899
+ 2021-11-08 15:57:35,322 INFO SenderThread:5242 [file_pusher.py:finish():177] shutting down file pusher
17900
+ 2021-11-08 15:57:35,322 INFO SenderThread:5242 [file_pusher.py:join():182] waiting for file pusher
17901
+ 2021-11-08 15:57:35,326 INFO MainThread:5242 [internal.py:handle_exit():77] Internal process exited
wandb/run-20211108_082107-2idnuxcc/logs/debug.log CHANGED
@@ -23,3 +23,109 @@ config: {}
23
  2021-11-08 08:21:10,483 INFO MainThread:3923 [wandb_run.py:_config_callback():911] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': True, 'do_eval': True, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 2, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 5e-05, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 3.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 0, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Nov08_08-20-06_t1v-n-ba7aa1a5-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 1000, 'logging_nan_inf_filter': True, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 25000, 'save_total_limit': 1, 'save_on_each_node': False, 'no_cuda': False, 'seed': 18, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'xpu_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 5000, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': 'en-ta-hi-te-from-100k', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': 'checkpoint_100000', 'hub_model_id': None, 'hub_strategy': 'HubStrategy.EVERY_SAVE', 'hub_token': None, 'gradient_checkpointing': False, 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': '', '_n_gpu': 0, '__cached__setup_devices': 'cpu'}
24
  2021-11-08 08:21:10,485 INFO MainThread:3923 [wandb_run.py:_config_callback():911] config_cb None None {'model_name_or_path': 'muril-bigbird', 'model_type': None, 'config_name': None, 'tokenizer_name': 'muril-bigbird', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'bfloat16'}
25
  2021-11-08 08:21:10,487 INFO MainThread:3923 [wandb_run.py:_config_callback():911] config_cb None None {'dataset_name': None, 'pretokenized': True, 'streaming': False, 'dataset_config_name': None, 'train_file': '/dev/shm/data-files', 'validation_file': '.', 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': None, 'mlm_probability': 0.15, 'pad_to_max_length': True, 'line_by_line': False}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
23
  2021-11-08 08:21:10,483 INFO MainThread:3923 [wandb_run.py:_config_callback():911] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': True, 'do_eval': True, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 2, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 5e-05, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 3.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 0, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Nov08_08-20-06_t1v-n-ba7aa1a5-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 1000, 'logging_nan_inf_filter': True, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 25000, 'save_total_limit': 1, 'save_on_each_node': False, 'no_cuda': False, 'seed': 18, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'xpu_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 5000, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': 'en-ta-hi-te-from-100k', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': 'checkpoint_100000', 'hub_model_id': None, 'hub_strategy': 'HubStrategy.EVERY_SAVE', 'hub_token': None, 'gradient_checkpointing': False, 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': '', '_n_gpu': 0, '__cached__setup_devices': 'cpu'}
24
  2021-11-08 08:21:10,485 INFO MainThread:3923 [wandb_run.py:_config_callback():911] config_cb None None {'model_name_or_path': 'muril-bigbird', 'model_type': None, 'config_name': None, 'tokenizer_name': 'muril-bigbird', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'bfloat16'}
25
  2021-11-08 08:21:10,487 INFO MainThread:3923 [wandb_run.py:_config_callback():911] config_cb None None {'dataset_name': None, 'pretokenized': True, 'streaming': False, 'dataset_config_name': None, 'train_file': '/dev/shm/data-files', 'validation_file': '.', 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': None, 'mlm_probability': 0.15, 'pad_to_max_length': True, 'line_by_line': False}
26
+ 2021-11-08 15:57:29,257 INFO MainThread:3923 [wandb_run.py:_atexit_cleanup():1663] got exitcode: 255
27
+ 2021-11-08 15:57:29,257 INFO MainThread:3923 [wandb_run.py:_restore():1635] restore
28
+ 2021-11-08 15:57:31,727 INFO MainThread:3923 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
29
+ wandb_count: 1
30
+ }
31
+ pusher_stats {
32
+ uploaded_bytes: 1825
33
+ total_bytes: 1825
34
+ }
35
+
36
+ 2021-11-08 15:57:31,985 INFO MainThread:3923 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
37
+ wandb_count: 1
38
+ }
39
+ pusher_stats {
40
+ uploaded_bytes: 1825
41
+ total_bytes: 1825
42
+ }
43
+
44
+ 2021-11-08 15:57:32,890 INFO MainThread:3923 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
45
+ wandb_count: 4
46
+ }
47
+ pusher_stats {
48
+ uploaded_bytes: 1825
49
+ total_bytes: 51125
50
+ }
51
+
52
+ 2021-11-08 15:57:32,992 INFO MainThread:3923 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
53
+ wandb_count: 5
54
+ }
55
+ pusher_stats {
56
+ uploaded_bytes: 1825
57
+ total_bytes: 51402
58
+ }
59
+
60
+ 2021-11-08 15:57:33,094 INFO MainThread:3923 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
61
+ wandb_count: 5
62
+ }
63
+ pusher_stats {
64
+ uploaded_bytes: 49366
65
+ total_bytes: 51402
66
+ }
67
+
68
+ 2021-11-08 15:57:33,196 INFO MainThread:3923 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
69
+ wandb_count: 5
70
+ }
71
+ pusher_stats {
72
+ uploaded_bytes: 51402
73
+ total_bytes: 51402
74
+ }
75
+
76
+ 2021-11-08 15:57:33,298 INFO MainThread:3923 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
77
+ wandb_count: 5
78
+ }
79
+ pusher_stats {
80
+ uploaded_bytes: 51402
81
+ total_bytes: 51402
82
+ }
83
+
84
+ 2021-11-08 15:57:33,400 INFO MainThread:3923 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
85
+ wandb_count: 5
86
+ }
87
+ pusher_stats {
88
+ uploaded_bytes: 51402
89
+ total_bytes: 51402
90
+ }
91
+
92
+ 2021-11-08 15:57:33,502 INFO MainThread:3923 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
93
+ wandb_count: 5
94
+ }
95
+ pusher_stats {
96
+ uploaded_bytes: 51402
97
+ total_bytes: 51402
98
+ }
99
+
100
+ 2021-11-08 15:57:33,603 INFO MainThread:3923 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
101
+ wandb_count: 5
102
+ }
103
+ pusher_stats {
104
+ uploaded_bytes: 51402
105
+ total_bytes: 51402
106
+ }
107
+
108
+ 2021-11-08 15:57:33,968 INFO MainThread:3923 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
109
+ wandb_count: 5
110
+ }
111
+ pusher_stats {
112
+ uploaded_bytes: 51402
113
+ total_bytes: 51402
114
+ }
115
+
116
+ 2021-11-08 15:57:34,323 INFO MainThread:3923 [wandb_run.py:_wait_for_finish():1793] got exit ret: done: true
117
+ exit_result {
118
+ }
119
+ file_counts {
120
+ wandb_count: 5
121
+ }
122
+ pusher_stats {
123
+ uploaded_bytes: 51402
124
+ total_bytes: 51402
125
+ }
126
+ local_info {
127
+ }
128
+
129
+ 2021-11-08 15:57:37,460 INFO MainThread:3923 [wandb_run.py:_append_history():2011] rendering history
130
+ 2021-11-08 15:57:37,460 INFO MainThread:3923 [wandb_run.py:_append_summary():1966] rendering summary
131
+ 2021-11-08 15:57:37,460 INFO MainThread:3923 [wandb_run.py:_append_files():2061] logging synced files
wandb/run-20211108_082107-2idnuxcc/run-2idnuxcc.wandb CHANGED
Binary files a/wandb/run-20211108_082107-2idnuxcc/run-2idnuxcc.wandb and b/wandb/run-20211108_082107-2idnuxcc/run-2idnuxcc.wandb differ
 
wandb/run-20211108_160143-33jzyy4p/files/config.yaml ADDED
@@ -0,0 +1,341 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ wandb_version: 1
2
+
3
+ __cached__setup_devices:
4
+ desc: null
5
+ value: cpu
6
+ _n_gpu:
7
+ desc: null
8
+ value: 0
9
+ _wandb:
10
+ desc: null
11
+ value:
12
+ cli_version: 0.12.6
13
+ framework: huggingface
14
+ huggingface_version: 4.13.0.dev0
15
+ is_jupyter_run: false
16
+ is_kaggle_kernel: true
17
+ python_version: 3.8.10
18
+ start_time: 1636387303
19
+ t:
20
+ 1:
21
+ - 1
22
+ - 2
23
+ - 3
24
+ - 11
25
+ - 12
26
+ 3:
27
+ - 13
28
+ 4: 3.8.10
29
+ 5: 0.12.6
30
+ 6: 4.13.0.dev0
31
+ 8:
32
+ - 2
33
+ - 5
34
+ adafactor:
35
+ desc: null
36
+ value: false
37
+ adam_beta1:
38
+ desc: null
39
+ value: 0.9
40
+ adam_beta2:
41
+ desc: null
42
+ value: 0.999
43
+ adam_epsilon:
44
+ desc: null
45
+ value: 1.0e-08
46
+ cache_dir:
47
+ desc: null
48
+ value: null
49
+ config_name:
50
+ desc: null
51
+ value: null
52
+ dataloader_drop_last:
53
+ desc: null
54
+ value: false
55
+ dataloader_num_workers:
56
+ desc: null
57
+ value: 0
58
+ dataloader_pin_memory:
59
+ desc: null
60
+ value: true
61
+ dataset_config_name:
62
+ desc: null
63
+ value: null
64
+ dataset_name:
65
+ desc: null
66
+ value: null
67
+ ddp_find_unused_parameters:
68
+ desc: null
69
+ value: null
70
+ debug:
71
+ desc: null
72
+ value: []
73
+ deepspeed:
74
+ desc: null
75
+ value: null
76
+ disable_tqdm:
77
+ desc: null
78
+ value: false
79
+ do_eval:
80
+ desc: null
81
+ value: true
82
+ do_predict:
83
+ desc: null
84
+ value: false
85
+ do_train:
86
+ desc: null
87
+ value: true
88
+ dtype:
89
+ desc: null
90
+ value: bfloat16
91
+ eval_accumulation_steps:
92
+ desc: null
93
+ value: null
94
+ eval_steps:
95
+ desc: null
96
+ value: 5000
97
+ evaluation_strategy:
98
+ desc: null
99
+ value: IntervalStrategy.NO
100
+ fp16:
101
+ desc: null
102
+ value: false
103
+ fp16_backend:
104
+ desc: null
105
+ value: auto
106
+ fp16_full_eval:
107
+ desc: null
108
+ value: false
109
+ fp16_opt_level:
110
+ desc: null
111
+ value: O1
112
+ gradient_accumulation_steps:
113
+ desc: null
114
+ value: 1
115
+ gradient_checkpointing:
116
+ desc: null
117
+ value: false
118
+ greater_is_better:
119
+ desc: null
120
+ value: null
121
+ group_by_length:
122
+ desc: null
123
+ value: false
124
+ hub_model_id:
125
+ desc: null
126
+ value: null
127
+ hub_strategy:
128
+ desc: null
129
+ value: HubStrategy.EVERY_SAVE
130
+ hub_token:
131
+ desc: null
132
+ value: null
133
+ ignore_data_skip:
134
+ desc: null
135
+ value: false
136
+ label_names:
137
+ desc: null
138
+ value: null
139
+ label_smoothing_factor:
140
+ desc: null
141
+ value: 0.0
142
+ learning_rate:
143
+ desc: null
144
+ value: 5.0e-05
145
+ length_column_name:
146
+ desc: null
147
+ value: length
148
+ line_by_line:
149
+ desc: null
150
+ value: false
151
+ load_best_model_at_end:
152
+ desc: null
153
+ value: false
154
+ local_rank:
155
+ desc: null
156
+ value: -1
157
+ log_level:
158
+ desc: null
159
+ value: -1
160
+ log_level_replica:
161
+ desc: null
162
+ value: -1
163
+ log_on_each_node:
164
+ desc: null
165
+ value: true
166
+ logging_dir:
167
+ desc: null
168
+ value: ./runs/Nov08_16-00-48_t1v-n-ba7aa1a5-w-0
169
+ logging_first_step:
170
+ desc: null
171
+ value: false
172
+ logging_nan_inf_filter:
173
+ desc: null
174
+ value: true
175
+ logging_steps:
176
+ desc: null
177
+ value: 1000
178
+ logging_strategy:
179
+ desc: null
180
+ value: IntervalStrategy.STEPS
181
+ lr_scheduler_type:
182
+ desc: null
183
+ value: SchedulerType.LINEAR
184
+ max_grad_norm:
185
+ desc: null
186
+ value: 1.0
187
+ max_seq_length:
188
+ desc: null
189
+ value: 4096
190
+ max_steps:
191
+ desc: null
192
+ value: -1
193
+ metric_for_best_model:
194
+ desc: null
195
+ value: null
196
+ mlm_probability:
197
+ desc: null
198
+ value: 0.15
199
+ model_name_or_path:
200
+ desc: null
201
+ value: muril-bigbird
202
+ model_type:
203
+ desc: null
204
+ value: null
205
+ mp_parameters:
206
+ desc: null
207
+ value: ''
208
+ no_cuda:
209
+ desc: null
210
+ value: false
211
+ num_train_epochs:
212
+ desc: null
213
+ value: 3.0
214
+ output_dir:
215
+ desc: null
216
+ value: ./
217
+ overwrite_cache:
218
+ desc: null
219
+ value: false
220
+ overwrite_output_dir:
221
+ desc: null
222
+ value: true
223
+ pad_to_max_length:
224
+ desc: null
225
+ value: true
226
+ past_index:
227
+ desc: null
228
+ value: -1
229
+ per_device_eval_batch_size:
230
+ desc: null
231
+ value: 2
232
+ per_device_train_batch_size:
233
+ desc: null
234
+ value: 1
235
+ per_gpu_eval_batch_size:
236
+ desc: null
237
+ value: null
238
+ per_gpu_train_batch_size:
239
+ desc: null
240
+ value: null
241
+ prediction_loss_only:
242
+ desc: null
243
+ value: false
244
+ preprocessing_num_workers:
245
+ desc: null
246
+ value: null
247
+ pretokenized:
248
+ desc: null
249
+ value: true
250
+ push_to_hub:
251
+ desc: null
252
+ value: true
253
+ push_to_hub_model_id:
254
+ desc: null
255
+ value: null
256
+ push_to_hub_organization:
257
+ desc: null
258
+ value: null
259
+ push_to_hub_token:
260
+ desc: null
261
+ value: null
262
+ remove_unused_columns:
263
+ desc: null
264
+ value: true
265
+ report_to:
266
+ desc: null
267
+ value:
268
+ - tensorboard
269
+ - wandb
270
+ resume_from_checkpoint:
271
+ desc: null
272
+ value: checkpoint_100000
273
+ run_name:
274
+ desc: null
275
+ value: en-ta-hi-te-from-150k
276
+ save_on_each_node:
277
+ desc: null
278
+ value: false
279
+ save_steps:
280
+ desc: null
281
+ value: 25000
282
+ save_strategy:
283
+ desc: null
284
+ value: IntervalStrategy.STEPS
285
+ save_total_limit:
286
+ desc: null
287
+ value: 1
288
+ seed:
289
+ desc: null
290
+ value: 18
291
+ sharded_ddp:
292
+ desc: null
293
+ value: []
294
+ skip_memory_metrics:
295
+ desc: null
296
+ value: true
297
+ streaming:
298
+ desc: null
299
+ value: false
300
+ tokenizer_name:
301
+ desc: null
302
+ value: muril-bigbird
303
+ tpu_metrics_debug:
304
+ desc: null
305
+ value: false
306
+ tpu_num_cores:
307
+ desc: null
308
+ value: null
309
+ train_file:
310
+ desc: null
311
+ value: /dev/shm/data-files
312
+ train_ref_file:
313
+ desc: null
314
+ value: null
315
+ use_fast_tokenizer:
316
+ desc: null
317
+ value: true
318
+ use_legacy_prediction_loop:
319
+ desc: null
320
+ value: false
321
+ validation_file:
322
+ desc: null
323
+ value: .
324
+ validation_ref_file:
325
+ desc: null
326
+ value: null
327
+ validation_split_percentage:
328
+ desc: null
329
+ value: 5
330
+ warmup_ratio:
331
+ desc: null
332
+ value: 0.0
333
+ warmup_steps:
334
+ desc: null
335
+ value: 0
336
+ weight_decay:
337
+ desc: null
338
+ value: 0.01
339
+ xpu_backend:
340
+ desc: null
341
+ value: null
wandb/run-20211108_160143-33jzyy4p/files/output.log ADDED
@@ -0,0 +1,6335 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ [16:02:02] - INFO - absl - A polynomial schedule was set with a non-positive `transition_steps` value; this results in a constant schedule with value `init_value`.
3
+ [16:02:02] - INFO - absl - Restoring checkpoint from checkpoint_100000
4
+ tcmalloc: large alloc 2886115328 bytes == 0x94a92000 @ 0x7ff058e15680 0x7ff058e36824 0x5bc8a7 0x50a83b 0x50c08b 0x5f68ef 0x57107c 0x56a0ba 0x5f6343 0x56bddd 0x56a0ba 0x68d5b7 0x67cd01 0x67cd7f 0x67ce21 0x67ef47 0x6b7242 0x6b75cd 0x7ff058c2c0b3 0x5fb18e
5
+ Epoch ... (1/3): 0%| | 0/3 [00:00<?, ?it/s]
6
+ Training...: 0%| | 0/59219 [00:00<?, ?it/s]
7
+
8
+
9
+
10
+
11
+
12
+
13
+
14
+
15
+
16
+
17
+
18
+
19
+
20
+
21
+
22
+
23
+
24
+
25
+
26
+
27
+
28
+
29
+
30
+
31
+
32
+
33
+
34
+
35
+
36
+
37
+
38
+
39
+
40
+
41
+
42
+
43
+
44
+
45
+
46
+
47
+
48
+
49
+
50
+
51
+
52
+
53
+
54
+
55
+
56
+
57
+
58
+
59
+
60
+
61
+
62
+
63
+
64
+
65
+
66
+
67
+
68
+
69
+
70
+
71
+
72
+
73
+
74
+
75
+
76
+
77
+
78
+
79
+
80
+
81
+
82
+
83
+
84
+
85
+
86
+
87
+
88
+
89
+
90
+
91
+
92
+
93
+
94
+
95
+
96
+
97
+
98
+
99
+
100
+
101
+
102
+
103
+
104
+
105
+
106
+
107
+
108
+
109
+
110
+
111
+
112
+
113
+
114
+
115
+
116
+
117
+
118
+
119
+
120
+
121
+
122
+
123
+
124
+
125
+
126
+
127
+
128
+
129
+
130
+
131
+
132
+
133
+
134
+
135
+
136
+
137
+
138
+
139
+
140
+
141
+
142
+
143
+
144
+
145
+
146
+
147
+
148
+
149
+
150
+
151
+
152
+
153
+
154
+
155
+
156
+
157
+
158
+
159
+
160
+
161
+
162
+
163
+
164
+
165
+
166
+
167
+
168
+
169
+
170
+
171
+
172
+
173
+
174
+
175
+
176
+
177
+
178
+
179
+
180
+
181
+
182
+
183
+
184
+
185
+
186
+
187
+
188
+
189
+
190
+
191
+
192
+
193
+
194
+
195
+
196
+
197
+
198
+
199
+
200
+
201
+
202
+
203
+
204
+
205
+
206
+
207
+
208
+
209
+
210
+
211
+
212
+
213
+
214
+
215
+
216
+
217
+
218
+
219
+
220
+
221
+
222
+
223
+
224
+
225
+
226
+
227
+
228
+
229
+
230
+
231
+
232
+
233
+
234
+
235
+
236
+
237
+
238
+
239
+
240
+
241
+ Training...: 2%|███▏ | 1012/59219 [10:20<6:32:05, 2.47it/s]
242
+
243
+
244
+
245
+
246
+
247
+
248
+
249
+
250
+
251
+
252
+
253
+
254
+
255
+
256
+
257
+
258
+
259
+
260
+
261
+
262
+
263
+
264
+
265
+
266
+
267
+
268
+
269
+
270
+
271
+
272
+
273
+
274
+
275
+
276
+
277
+
278
+
279
+
280
+
281
+
282
+
283
+
284
+
285
+
286
+
287
+
288
+
289
+
290
+
291
+
292
+
293
+
294
+
295
+
296
+
297
+
298
+
299
+
300
+
301
+
302
+
303
+
304
+
305
+
306
+
307
+
308
+
309
+
310
+
311
+
312
+
313
+
314
+
315
+
316
+
317
+
318
+
319
+
320
+
321
+
322
+
323
+
324
+
325
+
326
+
327
+
328
+
329
+
330
+
331
+
332
+
333
+
334
+
335
+
336
+
337
+
338
+
339
+
340
+
341
+
342
+
343
+
344
+
345
+
346
+
347
+
348
+
349
+
350
+
351
+
352
+
353
+
354
+
355
+
356
+
357
+
358
+
359
+
360
+
361
+
362
+
363
+
364
+
365
+
366
+
367
+
368
+
369
+
370
+
371
+
372
+
373
+
374
+
375
+
376
+
377
+
378
+
379
+
380
+
381
+
382
+
383
+
384
+
385
+
386
+
387
+
388
+
389
+
390
+
391
+
392
+
393
+
394
+
395
+
396
+
397
+
398
+
399
+
400
+
401
+
402
+
403
+
404
+
405
+
406
+
407
+
408
+
409
+
410
+
411
+
412
+
413
+
414
+
415
+
416
+
417
+
418
+
419
+
420
+
421
+
422
+
423
+
424
+
425
+
426
+
427
+
428
+
429
+
430
+
431
+
432
+
433
+
434
+
435
+
436
+
437
+
438
+
439
+
440
+
441
+
442
+
443
+
444
+
445
+
446
+
447
+
448
+
449
+
450
+
451
+
452
+
453
+
454
+
455
+
456
+
457
+
458
+
459
+
460
+
461
+
462
+
463
+
464
+
465
+
466
+
467
+
468
+
469
+
470
+
471
+
472
+
473
+
474
+
475
+
476
+ Training...: 3%|██████▎ | 1999/59219 [18:08<7:42:26, 2.06it/s]
477
+
478
+
479
+
480
+
481
+
482
+
483
+
484
+
485
+
486
+
487
+
488
+
489
+
490
+
491
+
492
+
493
+
494
+
495
+
496
+
497
+
498
+
499
+
500
+
501
+
502
+
503
+
504
+
505
+
506
+
507
+
508
+
509
+
510
+
511
+
512
+
513
+
514
+
515
+
516
+
517
+
518
+
519
+
520
+
521
+
522
+
523
+
524
+
525
+
526
+
527
+
528
+
529
+
530
+
531
+
532
+
533
+
534
+
535
+
536
+
537
+
538
+
539
+
540
+
541
+
542
+
543
+
544
+
545
+
546
+
547
+
548
+
549
+
550
+
551
+
552
+
553
+
554
+
555
+
556
+
557
+
558
+
559
+
560
+
561
+
562
+
563
+
564
+
565
+
566
+
567
+
568
+
569
+
570
+
571
+
572
+
573
+
574
+
575
+
576
+
577
+
578
+
579
+
580
+
581
+
582
+
583
+
584
+
585
+
586
+
587
+
588
+
589
+
590
+
591
+
592
+
593
+
594
+
595
+
596
+
597
+
598
+
599
+
600
+
601
+
602
+
603
+
604
+
605
+
606
+
607
+
608
+
609
+
610
+
611
+
612
+
613
+
614
+
615
+
616
+
617
+
618
+
619
+
620
+
621
+
622
+
623
+
624
+
625
+
626
+
627
+
628
+
629
+
630
+
631
+
632
+
633
+
634
+
635
+
636
+
637
+
638
+
639
+
640
+
641
+
642
+
643
+
644
+
645
+
646
+
647
+
648
+
649
+
650
+
651
+
652
+
653
+
654
+
655
+
656
+
657
+
658
+
659
+
660
+
661
+
662
+
663
+
664
+
665
+
666
+
667
+
668
+
669
+
670
+
671
+
672
+
673
+
674
+
675
+
676
+
677
+
678
+
679
+
680
+
681
+
682
+
683
+
684
+
685
+
686
+
687
+
688
+
689
+
690
+
691
+
692
+
693
+
694
+
695
+
696
+
697
+
698
+
699
+
700
+
701
+
702
+
703
+
704
+
705
+
706
+
707
+
708
+
709
+
710
+
711
+
712
+ Training...: 5%|█████████▌ | 2999/59219 [26:14<7:34:23, 2.06it/s]
713
+
714
+
715
+
716
+
717
+
718
+
719
+
720
+
721
+
722
+
723
+
724
+
725
+
726
+
727
+
728
+
729
+
730
+
731
+
732
+
733
+
734
+
735
+
736
+
737
+
738
+
739
+
740
+
741
+
742
+
743
+
744
+
745
+
746
+
747
+
748
+
749
+
750
+
751
+
752
+
753
+
754
+
755
+
756
+
757
+
758
+
759
+
760
+
761
+
762
+
763
+
764
+
765
+
766
+
767
+
768
+
769
+
770
+
771
+
772
+
773
+
774
+
775
+
776
+
777
+
778
+
779
+
780
+
781
+
782
+
783
+
784
+
785
+
786
+
787
+
788
+
789
+
790
+
791
+
792
+
793
+
794
+
795
+
796
+
797
+
798
+
799
+
800
+
801
+
802
+
803
+
804
+
805
+
806
+
807
+
808
+
809
+
810
+
811
+
812
+
813
+
814
+
815
+
816
+
817
+
818
+
819
+
820
+
821
+
822
+
823
+
824
+
825
+
826
+
827
+
828
+
829
+
830
+
831
+
832
+
833
+
834
+
835
+
836
+
837
+
838
+
839
+
840
+
841
+
842
+
843
+
844
+
845
+
846
+
847
+
848
+
849
+
850
+
851
+
852
+
853
+
854
+
855
+
856
+
857
+
858
+
859
+
860
+
861
+
862
+
863
+
864
+
865
+
866
+
867
+
868
+
869
+
870
+
871
+
872
+
873
+
874
+
875
+
876
+
877
+
878
+
879
+
880
+
881
+
882
+
883
+
884
+
885
+
886
+
887
+
888
+
889
+
890
+
891
+
892
+
893
+
894
+
895
+
896
+
897
+
898
+
899
+
900
+
901
+
902
+
903
+
904
+
905
+
906
+
907
+
908
+
909
+
910
+
911
+
912
+
913
+
914
+
915
+
916
+
917
+
918
+
919
+
920
+
921
+
922
+
923
+
924
+
925
+
926
+
927
+
928
+
929
+
930
+
931
+
932
+
933
+
934
+
935
+
936
+
937
+
938
+
939
+
940
+
941
+
942
+
943
+
944
+
945
+
946
+
947
+
948
+
949
+ Training...: 7%|████████████▋ | 4009/59219 [34:38<8:55:41, 1.72it/s]
950
+
951
+
952
+
953
+
954
+
955
+
956
+
957
+
958
+
959
+
960
+
961
+
962
+
963
+
964
+
965
+
966
+
967
+
968
+
969
+
970
+
971
+
972
+
973
+
974
+
975
+
976
+
977
+
978
+
979
+
980
+
981
+
982
+
983
+
984
+
985
+
986
+
987
+
988
+
989
+
990
+
991
+
992
+
993
+
994
+
995
+
996
+
997
+
998
+
999
+
1000
+
1001
+
1002
+
1003
+
1004
+
1005
+
1006
+
1007
+
1008
+
1009
+
1010
+
1011
+
1012
+
1013
+
1014
+
1015
+
1016
+
1017
+
1018
+
1019
+
1020
+
1021
+
1022
+
1023
+
1024
+
1025
+
1026
+
1027
+
1028
+
1029
+
1030
+
1031
+
1032
+
1033
+
1034
+
1035
+
1036
+
1037
+
1038
+
1039
+
1040
+
1041
+
1042
+
1043
+
1044
+
1045
+
1046
+
1047
+
1048
+
1049
+
1050
+
1051
+
1052
+
1053
+
1054
+
1055
+
1056
+
1057
+
1058
+
1059
+
1060
+
1061
+
1062
+
1063
+
1064
+
1065
+
1066
+
1067
+
1068
+
1069
+
1070
+
1071
+
1072
+
1073
+
1074
+
1075
+
1076
+
1077
+
1078
+
1079
+
1080
+
1081
+
1082
+
1083
+
1084
+
1085
+
1086
+
1087
+
1088
+
1089
+
1090
+
1091
+
1092
+
1093
+
1094
+
1095
+
1096
+
1097
+
1098
+
1099
+
1100
+
1101
+
1102
+
1103
+
1104
+
1105
+
1106
+
1107
+
1108
+
1109
+
1110
+
1111
+
1112
+
1113
+
1114
+
1115
+
1116
+
1117
+
1118
+
1119
+
1120
+
1121
+
1122
+
1123
+
1124
+
1125
+
1126
+
1127
+
1128
+
1129
+
1130
+
1131
+
1132
+
1133
+
1134
+
1135
+
1136
+
1137
+
1138
+
1139
+
1140
+
1141
+
1142
+
1143
+
1144
+
1145
+
1146
+
1147
+
1148
+
1149
+
1150
+
1151
+
1152
+
1153
+
1154
+
1155
+
1156
+
1157
+
1158
+
1159
+
1160
+
1161
+
1162
+
1163
+
1164
+
1165
+
1166
+
1167
+
1168
+
1169
+
1170
+
1171
+
1172
+
1173
+
1174
+
1175
+
1176
+
1177
+
1178
+
1179
+
1180
+
1181
+
1182
+
1183
+
1184
+
1185
+ Training...: 8%|███████████████▊ | 4999/59219 [42:44<7:18:18, 2.06it/s]
1186
+ Step... (105000 | Loss: 1.3476334810256958, Learning Rate: 2.0448671421036124e-05)
1187
+
1188
+
1189
+
1190
+
1191
+
1192
+
1193
+
1194
+
1195
+
1196
+
1197
+
1198
+
1199
+
1200
+
1201
+
1202
+
1203
+
1204
+
1205
+
1206
+
1207
+
1208
+
1209
+
1210
+
1211
+
1212
+
1213
+
1214
+
1215
+
1216
+
1217
+
1218
+
1219
+
1220
+
1221
+
1222
+
1223
+
1224
+
1225
+
1226
+
1227
+
1228
+
1229
+
1230
+
1231
+
1232
+
1233
+
1234
+
1235
+
1236
+
1237
+
1238
+
1239
+
1240
+
1241
+
1242
+
1243
+
1244
+
1245
+
1246
+
1247
+
1248
+
1249
+
1250
+
1251
+
1252
+
1253
+
1254
+
1255
+
1256
+
1257
+
1258
+
1259
+
1260
+
1261
+
1262
+
1263
+
1264
+
1265
+
1266
+
1267
+
1268
+
1269
+
1270
+
1271
+
1272
+
1273
+
1274
+
1275
+
1276
+
1277
+
1278
+
1279
+
1280
+
1281
+
1282
+
1283
+
1284
+
1285
+
1286
+
1287
+
1288
+
1289
+
1290
+
1291
+
1292
+
1293
+
1294
+
1295
+
1296
+
1297
+
1298
+
1299
+
1300
+
1301
+
1302
+
1303
+
1304
+
1305
+
1306
+
1307
+
1308
+
1309
+
1310
+
1311
+
1312
+
1313
+
1314
+
1315
+
1316
+
1317
+
1318
+
1319
+
1320
+
1321
+
1322
+
1323
+
1324
+
1325
+
1326
+
1327
+
1328
+
1329
+
1330
+
1331
+
1332
+
1333
+
1334
+
1335
+
1336
+
1337
+
1338
+
1339
+
1340
+
1341
+
1342
+
1343
+
1344
+
1345
+
1346
+
1347
+
1348
+
1349
+
1350
+
1351
+
1352
+
1353
+
1354
+
1355
+
1356
+
1357
+
1358
+
1359
+
1360
+
1361
+
1362
+
1363
+
1364
+
1365
+
1366
+
1367
+
1368
+
1369
+
1370
+
1371
+
1372
+
1373
+
1374
+
1375
+
1376
+
1377
+
1378
+
1379
+
1380
+
1381
+
1382
+
1383
+
1384
+
1385
+
1386
+
1387
+
1388
+
1389
+
1390
+
1391
+
1392
+
1393
+
1394
+
1395
+
1396
+
1397
+
1398
+
1399
+
1400
+
1401
+
1402
+
1403
+
1404
+
1405
+
1406
+
1407
+
1408
+
1409
+
1410
+
1411
+
1412
+
1413
+
1414
+
1415
+
1416
+
1417
+
1418
+
1419
+
1420
+
1421
+
1422
+
1423
+
1424
+
1425
+
1426
+
1427
+
1428
+
1429
+
1430
+
1431
+
1432
+
1433
+
1434
+
1435
+
1436
+
1437
+
1438
+
1439
+
1440
+
1441
+
1442
+
1443
+
1444
+
1445
+
1446
+
1447
+
1448
+
1449
+
1450
+
1451
+
1452
+
1453
+
1454
+
1455
+
1456
+
1457
+
1458
+
1459
+
1460
+
1461
+
1462
+
1463
+
1464
+
1465
+
1466
+
1467
+
1468
+
1469
+
1470
+
1471
+
1472
+
1473
+
1474
+
1475
+
1476
+
1477
+
1478
+
1479
+
1480
+
1481
+
1482
+
1483
+
1484
+
1485
+
1486
+
1487
+
1488
+
1489
+
1490
+
1491
+
1492
+
1493
+
1494
+
1495
+
1496
+
1497
+
1498
+
1499
+
1500
+
1501
+
1502
+
1503
+
1504
+
1505
+
1506
+
1507
+
1508
+ Training...: 10%|███████████████████ | 6010/59219 [54:15<8:17:08, 1.78it/s]
1509
+
1510
+
1511
+
1512
+
1513
+
1514
+
1515
+
1516
+
1517
+
1518
+
1519
+
1520
+
1521
+
1522
+
1523
+
1524
+
1525
+
1526
+
1527
+
1528
+
1529
+
1530
+
1531
+
1532
+
1533
+
1534
+
1535
+
1536
+
1537
+
1538
+
1539
+
1540
+
1541
+
1542
+
1543
+
1544
+
1545
+
1546
+
1547
+
1548
+
1549
+
1550
+
1551
+
1552
+
1553
+
1554
+
1555
+
1556
+
1557
+
1558
+
1559
+
1560
+
1561
+
1562
+
1563
+
1564
+
1565
+
1566
+
1567
+
1568
+
1569
+
1570
+
1571
+
1572
+
1573
+
1574
+
1575
+
1576
+
1577
+
1578
+
1579
+
1580
+
1581
+
1582
+
1583
+
1584
+
1585
+
1586
+
1587
+
1588
+
1589
+
1590
+
1591
+
1592
+
1593
+
1594
+
1595
+
1596
+
1597
+
1598
+
1599
+
1600
+
1601
+
1602
+
1603
+
1604
+
1605
+
1606
+
1607
+
1608
+
1609
+
1610
+
1611
+
1612
+
1613
+
1614
+
1615
+
1616
+
1617
+
1618
+
1619
+
1620
+
1621
+
1622
+
1623
+
1624
+
1625
+
1626
+
1627
+
1628
+
1629
+
1630
+
1631
+
1632
+
1633
+
1634
+
1635
+
1636
+
1637
+
1638
+
1639
+
1640
+
1641
+
1642
+
1643
+
1644
+
1645
+
1646
+
1647
+
1648
+
1649
+
1650
+
1651
+
1652
+
1653
+
1654
+
1655
+
1656
+
1657
+
1658
+
1659
+
1660
+
1661
+
1662
+
1663
+
1664
+
1665
+
1666
+
1667
+
1668
+
1669
+
1670
+
1671
+
1672
+
1673
+
1674
+
1675
+
1676
+
1677
+
1678
+
1679
+
1680
+
1681
+
1682
+
1683
+
1684
+
1685
+
1686
+
1687
+
1688
+
1689
+
1690
+
1691
+
1692
+
1693
+
1694
+
1695
+
1696
+
1697
+
1698
+
1699
+
1700
+
1701
+
1702
+
1703
+
1704
+
1705
+
1706
+
1707
+
1708
+
1709
+
1710
+
1711
+
1712
+
1713
+
1714
+
1715
+
1716
+
1717
+
1718
+
1719
+
1720
+
1721
+
1722
+
1723
+
1724
+
1725
+
1726
+
1727
+
1728
+
1729
+
1730
+
1731
+
1732
+
1733
+
1734
+
1735
+
1736
+
1737
+
1738
+
1739
+
1740
+
1741
+
1742
+
1743
+
1744
+ Training...: 12%|█████████████████████▉ | 7004/59219 [1:02:21<26:21:36, 1.82s/it]
1745
+
1746
+
1747
+
1748
+
1749
+
1750
+
1751
+
1752
+
1753
+
1754
+
1755
+
1756
+
1757
+
1758
+
1759
+
1760
+
1761
+
1762
+
1763
+
1764
+
1765
+
1766
+
1767
+
1768
+
1769
+
1770
+
1771
+
1772
+
1773
+
1774
+
1775
+
1776
+
1777
+
1778
+
1779
+
1780
+
1781
+
1782
+
1783
+
1784
+
1785
+
1786
+
1787
+
1788
+
1789
+
1790
+
1791
+
1792
+
1793
+
1794
+
1795
+
1796
+
1797
+
1798
+
1799
+
1800
+
1801
+
1802
+
1803
+
1804
+
1805
+
1806
+
1807
+
1808
+
1809
+
1810
+
1811
+
1812
+
1813
+
1814
+
1815
+
1816
+
1817
+
1818
+
1819
+
1820
+
1821
+
1822
+
1823
+
1824
+
1825
+
1826
+
1827
+
1828
+
1829
+
1830
+
1831
+
1832
+
1833
+
1834
+
1835
+
1836
+
1837
+
1838
+
1839
+
1840
+
1841
+
1842
+
1843
+
1844
+
1845
+
1846
+
1847
+
1848
+
1849
+
1850
+
1851
+
1852
+
1853
+
1854
+
1855
+
1856
+
1857
+
1858
+
1859
+
1860
+
1861
+
1862
+
1863
+
1864
+
1865
+
1866
+
1867
+
1868
+
1869
+
1870
+
1871
+
1872
+
1873
+
1874
+
1875
+
1876
+
1877
+
1878
+
1879
+
1880
+
1881
+
1882
+
1883
+
1884
+
1885
+
1886
+
1887
+
1888
+
1889
+
1890
+
1891
+
1892
+
1893
+
1894
+
1895
+
1896
+
1897
+
1898
+
1899
+
1900
+
1901
+
1902
+
1903
+
1904
+
1905
+
1906
+
1907
+
1908
+
1909
+
1910
+
1911
+
1912
+
1913
+
1914
+
1915
+
1916
+
1917
+
1918
+
1919
+
1920
+
1921
+
1922
+
1923
+
1924
+
1925
+
1926
+
1927
+
1928
+
1929
+
1930
+
1931
+
1932
+
1933
+
1934
+
1935
+
1936
+
1937
+
1938
+
1939
+
1940
+
1941
+
1942
+
1943
+
1944
+
1945
+
1946
+
1947
+
1948
+
1949
+
1950
+
1951
+
1952
+
1953
+
1954
+
1955
+
1956
+
1957
+
1958
+
1959
+
1960
+
1961
+
1962
+
1963
+
1964
+
1965
+
1966
+
1967
+
1968
+
1969
+
1970
+
1971
+
1972
+
1973
+
1974
+
1975
+
1976
+
1977
+
1978
+
1979
+
1980
+ Training...: 14%|████████████████████████▉ | 8000/59219 [1:10:27<77:33:58, 5.45s/it]
1981
+
1982
+
1983
+
1984
+
1985
+
1986
+
1987
+
1988
+
1989
+
1990
+
1991
+
1992
+
1993
+
1994
+
1995
+
1996
+
1997
+
1998
+
1999
+
2000
+
2001
+
2002
+
2003
+
2004
+
2005
+
2006
+
2007
+
2008
+
2009
+
2010
+
2011
+
2012
+
2013
+
2014
+
2015
+
2016
+
2017
+
2018
+
2019
+
2020
+
2021
+
2022
+
2023
+
2024
+
2025
+
2026
+
2027
+
2028
+
2029
+
2030
+
2031
+
2032
+
2033
+
2034
+
2035
+
2036
+
2037
+
2038
+
2039
+
2040
+
2041
+
2042
+
2043
+
2044
+
2045
+
2046
+
2047
+
2048
+
2049
+
2050
+
2051
+
2052
+
2053
+
2054
+
2055
+
2056
+
2057
+
2058
+
2059
+
2060
+
2061
+
2062
+
2063
+
2064
+
2065
+
2066
+
2067
+
2068
+
2069
+
2070
+
2071
+
2072
+
2073
+
2074
+
2075
+
2076
+
2077
+
2078
+
2079
+
2080
+
2081
+
2082
+
2083
+
2084
+
2085
+
2086
+
2087
+
2088
+
2089
+
2090
+
2091
+
2092
+
2093
+
2094
+
2095
+
2096
+
2097
+
2098
+
2099
+
2100
+
2101
+
2102
+
2103
+
2104
+
2105
+
2106
+
2107
+
2108
+
2109
+
2110
+
2111
+
2112
+
2113
+
2114
+
2115
+
2116
+
2117
+
2118
+
2119
+
2120
+
2121
+
2122
+
2123
+
2124
+
2125
+
2126
+
2127
+
2128
+
2129
+
2130
+
2131
+
2132
+
2133
+
2134
+
2135
+
2136
+
2137
+
2138
+
2139
+
2140
+
2141
+
2142
+
2143
+
2144
+
2145
+
2146
+
2147
+
2148
+
2149
+
2150
+
2151
+
2152
+
2153
+
2154
+
2155
+
2156
+
2157
+
2158
+
2159
+
2160
+
2161
+
2162
+
2163
+
2164
+
2165
+
2166
+
2167
+
2168
+
2169
+
2170
+
2171
+
2172
+
2173
+
2174
+
2175
+
2176
+
2177
+
2178
+
2179
+
2180
+
2181
+
2182
+
2183
+
2184
+
2185
+
2186
+
2187
+
2188
+
2189
+
2190
+
2191
+
2192
+
2193
+
2194
+
2195
+
2196
+
2197
+
2198
+
2199
+
2200
+
2201
+
2202
+
2203
+
2204
+
2205
+
2206
+
2207
+
2208
+
2209
+
2210
+
2211
+
2212
+
2213
+
2214
+
2215
+
2216
+ Training...: 15%|████████████████████████████▎ | 8999/59219 [1:18:16<6:46:00, 2.06it/s]
2217
+ Step... (109000 | Loss: 1.5966984033584595, Learning Rate: 1.932290797412861e-05)
2218
+ Step... (105000 | Loss: 1.2561148405075073, Acc: 0.7473720908164978): 0%| | 0/3 [1:19:36<?, ?it/s]
2219
+
2220
+
2221
+
2222
+
2223
+
2224
+
2225
+
2226
+
2227
+
2228
+
2229
+
2230
+
2231
+
2232
+
2233
+
2234
+
2235
+
2236
+
2237
+
2238
+
2239
+
2240
+
2241
+
2242
+
2243
+
2244
+
2245
+
2246
+
2247
+
2248
+
2249
+
2250
+
2251
+
2252
+
2253
+
2254
+
2255
+
2256
+
2257
+
2258
+
2259
+
2260
+
2261
+
2262
+
2263
+
2264
+
2265
+
2266
+
2267
+
2268
+
2269
+
2270
+
2271
+
2272
+
2273
+
2274
+
2275
+
2276
+
2277
+
2278
+
2279
+
2280
+
2281
+
2282
+
2283
+
2284
+
2285
+
2286
+
2287
+
2288
+
2289
+
2290
+
2291
+
2292
+
2293
+
2294
+
2295
+
2296
+
2297
+
2298
+
2299
+
2300
+
2301
+
2302
+
2303
+
2304
+
2305
+
2306
+
2307
+
2308
+
2309
+
2310
+
2311
+
2312
+
2313
+
2314
+
2315
+
2316
+
2317
+
2318
+
2319
+
2320
+
2321
+
2322
+
2323
+
2324
+
2325
+
2326
+
2327
+
2328
+
2329
+
2330
+
2331
+
2332
+
2333
+
2334
+
2335
+
2336
+
2337
+
2338
+
2339
+
2340
+
2341
+
2342
+
2343
+
2344
+
2345
+
2346
+
2347
+
2348
+
2349
+
2350
+
2351
+
2352
+
2353
+
2354
+
2355
+
2356
+
2357
+
2358
+
2359
+
2360
+
2361
+
2362
+
2363
+
2364
+
2365
+
2366
+
2367
+
2368
+
2369
+
2370
+
2371
+
2372
+
2373
+
2374
+
2375
+
2376
+
2377
+
2378
+
2379
+
2380
+
2381
+
2382
+
2383
+
2384
+
2385
+
2386
+
2387
+
2388
+
2389
+
2390
+
2391
+
2392
+
2393
+
2394
+
2395
+
2396
+
2397
+
2398
+
2399
+
2400
+
2401
+
2402
+
2403
+
2404
+
2405
+
2406
+
2407
+
2408
+
2409
+
2410
+
2411
+
2412
+
2413
+
2414
+
2415
+
2416
+
2417
+
2418
+
2419
+
2420
+
2421
+
2422
+
2423
+
2424
+
2425
+
2426
+
2427
+
2428
+
2429
+
2430
+
2431
+
2432
+
2433
+
2434
+
2435
+
2436
+
2437
+
2438
+
2439
+
2440
+
2441
+
2442
+
2443
+
2444
+
2445
+
2446
+
2447
+
2448
+
2449
+
2450
+
2451
+
2452
+ Training...: 17%|███████████████████████████████▍ | 9999/59219 [1:26:22<6:38:06, 2.06it/s]
2453
+ Training...: 17%|███████████████████████████████▍ | 9999/59219 [1:26:39<6:38:06, 2.06it/s]
2454
+
2455
+
2456
+
2457
+
2458
+
2459
+
2460
+
2461
+
2462
+
2463
+
2464
+
2465
+
2466
+
2467
+
2468
+
2469
+
2470
+
2471
+
2472
+
2473
+
2474
+
2475
+
2476
+
2477
+
2478
+
2479
+
2480
+
2481
+
2482
+
2483
+
2484
+
2485
+
2486
+
2487
+
2488
+
2489
+
2490
+
2491
+
2492
+
2493
+
2494
+
2495
+
2496
+
2497
+
2498
+
2499
+
2500
+
2501
+
2502
+
2503
+
2504
+
2505
+
2506
+
2507
+
2508
+
2509
+
2510
+
2511
+
2512
+
2513
+
2514
+
2515
+
2516
+
2517
+
2518
+
2519
+
2520
+
2521
+
2522
+
2523
+
2524
+
2525
+
2526
+
2527
+
2528
+
2529
+
2530
+
2531
+
2532
+
2533
+
2534
+
2535
+
2536
+
2537
+
2538
+
2539
+
2540
+
2541
+
2542
+
2543
+
2544
+
2545
+
2546
+
2547
+
2548
+
2549
+
2550
+
2551
+
2552
+
2553
+
2554
+
2555
+
2556
+
2557
+
2558
+
2559
+
2560
+
2561
+
2562
+
2563
+
2564
+
2565
+
2566
+
2567
+
2568
+
2569
+
2570
+
2571
+
2572
+
2573
+
2574
+
2575
+
2576
+
2577
+
2578
+
2579
+
2580
+
2581
+
2582
+
2583
+
2584
+
2585
+
2586
+
2587
+
2588
+
2589
+
2590
+
2591
+
2592
+
2593
+
2594
+
2595
+
2596
+
2597
+
2598
+
2599
+
2600
+
2601
+
2602
+
2603
+
2604
+
2605
+
2606
+
2607
+
2608
+
2609
+
2610
+
2611
+
2612
+
2613
+
2614
+
2615
+
2616
+
2617
+
2618
+
2619
+
2620
+
2621
+
2622
+
2623
+
2624
+
2625
+
2626
+
2627
+
2628
+
2629
+
2630
+
2631
+
2632
+
2633
+
2634
+
2635
+
2636
+
2637
+
2638
+
2639
+
2640
+
2641
+
2642
+
2643
+
2644
+
2645
+
2646
+
2647
+
2648
+
2649
+
2650
+
2651
+
2652
+
2653
+
2654
+
2655
+
2656
+
2657
+
2658
+
2659
+
2660
+
2661
+
2662
+
2663
+
2664
+
2665
+
2666
+
2667
+
2668
+
2669
+
2670
+
2671
+
2672
+
2673
+
2674
+
2675
+
2676
+
2677
+
2678
+
2679
+
2680
+
2681
+
2682
+
2683
+
2684
+
2685
+
2686
+
2687
+
2688
+
2689
+
2690
+
2691
+
2692
+
2693
+
2694
+
2695
+
2696
+
2697
+
2698
+
2699
+
2700
+
2701
+
2702
+
2703
+
2704
+
2705
+
2706
+
2707
+
2708
+
2709
+
2710
+
2711
+
2712
+
2713
+
2714
+
2715
+
2716
+
2717
+
2718
+
2719
+
2720
+
2721
+
2722
+
2723
+
2724
+
2725
+
2726
+
2727
+
2728
+
2729
+
2730
+
2731
+
2732
+
2733
+
2734
+
2735
+
2736
+
2737
+
2738
+
2739
+
2740
+
2741
+
2742
+
2743
+
2744
+
2745
+
2746
+
2747
+
2748
+
2749
+
2750
+
2751
+
2752
+
2753
+
2754
+
2755
+
2756
+
2757
+
2758
+
2759
+
2760
+
2761
+
2762
+
2763
+
2764
+
2765
+
2766
+
2767
+
2768
+
2769
+
2770
+
2771
+
2772
+
2773
+
2774
+ Training...: 19%|██████████████████████████████████▎ | 10999/59219 [1:37:30<6:29:05, 2.07it/s]4it/s]
2775
+
2776
+
2777
+
2778
+
2779
+
2780
+
2781
+
2782
+
2783
+
2784
+
2785
+
2786
+
2787
+
2788
+
2789
+
2790
+
2791
+
2792
+
2793
+
2794
+
2795
+
2796
+
2797
+
2798
+
2799
+
2800
+
2801
+
2802
+
2803
+
2804
+
2805
+
2806
+
2807
+
2808
+
2809
+
2810
+
2811
+
2812
+
2813
+
2814
+
2815
+
2816
+
2817
+
2818
+
2819
+
2820
+
2821
+
2822
+
2823
+
2824
+
2825
+
2826
+
2827
+
2828
+
2829
+
2830
+
2831
+
2832
+
2833
+
2834
+
2835
+
2836
+
2837
+
2838
+
2839
+
2840
+
2841
+
2842
+
2843
+
2844
+
2845
+
2846
+
2847
+
2848
+
2849
+
2850
+
2851
+
2852
+
2853
+
2854
+
2855
+
2856
+
2857
+
2858
+
2859
+
2860
+
2861
+
2862
+
2863
+
2864
+
2865
+
2866
+
2867
+
2868
+
2869
+
2870
+
2871
+
2872
+
2873
+
2874
+
2875
+
2876
+
2877
+
2878
+
2879
+
2880
+
2881
+
2882
+
2883
+
2884
+
2885
+
2886
+
2887
+
2888
+
2889
+
2890
+
2891
+
2892
+
2893
+
2894
+
2895
+
2896
+
2897
+
2898
+
2899
+
2900
+
2901
+
2902
+
2903
+
2904
+
2905
+
2906
+
2907
+
2908
+
2909
+
2910
+
2911
+
2912
+
2913
+
2914
+
2915
+
2916
+
2917
+
2918
+
2919
+
2920
+
2921
+
2922
+
2923
+
2924
+
2925
+
2926
+
2927
+
2928
+
2929
+
2930
+
2931
+
2932
+
2933
+
2934
+
2935
+
2936
+
2937
+
2938
+
2939
+
2940
+
2941
+
2942
+
2943
+
2944
+
2945
+
2946
+
2947
+
2948
+
2949
+
2950
+
2951
+
2952
+
2953
+
2954
+
2955
+
2956
+
2957
+
2958
+
2959
+
2960
+
2961
+
2962
+
2963
+
2964
+
2965
+
2966
+
2967
+
2968
+
2969
+
2970
+
2971
+
2972
+
2973
+
2974
+
2975
+
2976
+
2977
+
2978
+
2979
+
2980
+
2981
+
2982
+
2983
+
2984
+
2985
+
2986
+
2987
+
2988
+
2989
+
2990
+
2991
+
2992
+
2993
+
2994
+
2995
+
2996
+
2997
+
2998
+
2999
+
3000
+
3001
+
3002
+
3003
+
3004
+
3005
+
3006
+
3007
+
3008
+
3009
+
3010
+ Training...: 20%|█████████████████████████████████████▌ | 12022/59219 [1:45:54<1:33:59, 8.37it/s]4it/s]
3011
+
3012
+
3013
+
3014
+
3015
+
3016
+
3017
+
3018
+
3019
+
3020
+
3021
+
3022
+
3023
+
3024
+
3025
+
3026
+
3027
+
3028
+
3029
+
3030
+
3031
+
3032
+
3033
+
3034
+
3035
+
3036
+
3037
+
3038
+
3039
+
3040
+
3041
+
3042
+
3043
+
3044
+
3045
+
3046
+
3047
+
3048
+
3049
+
3050
+
3051
+
3052
+
3053
+
3054
+
3055
+
3056
+
3057
+
3058
+
3059
+
3060
+
3061
+
3062
+
3063
+
3064
+
3065
+
3066
+
3067
+
3068
+
3069
+
3070
+
3071
+
3072
+
3073
+
3074
+
3075
+
3076
+
3077
+
3078
+
3079
+
3080
+
3081
+
3082
+
3083
+
3084
+
3085
+
3086
+
3087
+
3088
+
3089
+
3090
+
3091
+
3092
+
3093
+
3094
+
3095
+
3096
+
3097
+
3098
+
3099
+
3100
+
3101
+
3102
+
3103
+
3104
+
3105
+
3106
+
3107
+
3108
+
3109
+
3110
+
3111
+
3112
+
3113
+
3114
+
3115
+
3116
+
3117
+
3118
+
3119
+
3120
+
3121
+
3122
+
3123
+
3124
+
3125
+
3126
+
3127
+
3128
+
3129
+
3130
+
3131
+
3132
+
3133
+
3134
+
3135
+
3136
+
3137
+
3138
+
3139
+
3140
+
3141
+
3142
+
3143
+
3144
+
3145
+
3146
+
3147
+
3148
+
3149
+
3150
+
3151
+
3152
+
3153
+
3154
+
3155
+
3156
+
3157
+
3158
+
3159
+
3160
+
3161
+
3162
+
3163
+
3164
+
3165
+
3166
+
3167
+
3168
+
3169
+
3170
+
3171
+
3172
+
3173
+
3174
+
3175
+
3176
+
3177
+
3178
+
3179
+
3180
+
3181
+
3182
+
3183
+
3184
+
3185
+
3186
+
3187
+
3188
+
3189
+
3190
+
3191
+
3192
+
3193
+
3194
+
3195
+
3196
+
3197
+
3198
+
3199
+
3200
+
3201
+
3202
+
3203
+
3204
+
3205
+
3206
+
3207
+
3208
+
3209
+
3210
+
3211
+
3212
+
3213
+
3214
+
3215
+
3216
+
3217
+
3218
+
3219
+
3220
+
3221
+
3222
+
3223
+
3224
+
3225
+
3226
+
3227
+
3228
+
3229
+
3230
+
3231
+
3232
+
3233
+
3234
+
3235
+
3236
+
3237
+
3238
+
3239
+
3240
+
3241
+
3242
+
3243
+
3244
+ Training...: 22%|████████████████████████████████████████▌ | 12999/59219 [1:53:42<6:13:36, 2.06it/s]4it/s]
3245
+
3246
+
3247
+
3248
+
3249
+
3250
+
3251
+
3252
+
3253
+
3254
+
3255
+
3256
+
3257
+
3258
+
3259
+
3260
+
3261
+
3262
+
3263
+
3264
+
3265
+
3266
+
3267
+
3268
+
3269
+
3270
+
3271
+
3272
+
3273
+
3274
+
3275
+
3276
+
3277
+
3278
+
3279
+
3280
+
3281
+
3282
+
3283
+
3284
+
3285
+
3286
+
3287
+
3288
+
3289
+
3290
+
3291
+
3292
+
3293
+
3294
+
3295
+
3296
+
3297
+
3298
+
3299
+
3300
+
3301
+
3302
+
3303
+
3304
+
3305
+
3306
+
3307
+
3308
+
3309
+
3310
+
3311
+
3312
+
3313
+
3314
+
3315
+
3316
+
3317
+
3318
+
3319
+
3320
+
3321
+
3322
+
3323
+
3324
+
3325
+
3326
+
3327
+
3328
+
3329
+
3330
+
3331
+
3332
+
3333
+
3334
+
3335
+
3336
+
3337
+
3338
+
3339
+
3340
+
3341
+
3342
+
3343
+
3344
+
3345
+
3346
+
3347
+
3348
+
3349
+
3350
+
3351
+
3352
+
3353
+
3354
+
3355
+
3356
+
3357
+
3358
+
3359
+
3360
+
3361
+
3362
+
3363
+
3364
+
3365
+
3366
+
3367
+
3368
+
3369
+
3370
+
3371
+
3372
+
3373
+
3374
+
3375
+
3376
+
3377
+
3378
+
3379
+
3380
+
3381
+
3382
+
3383
+
3384
+
3385
+
3386
+
3387
+
3388
+
3389
+
3390
+
3391
+
3392
+
3393
+
3394
+
3395
+
3396
+
3397
+
3398
+
3399
+
3400
+
3401
+
3402
+
3403
+
3404
+
3405
+
3406
+
3407
+
3408
+
3409
+
3410
+
3411
+
3412
+
3413
+
3414
+
3415
+
3416
+
3417
+
3418
+
3419
+
3420
+
3421
+
3422
+
3423
+
3424
+
3425
+
3426
+
3427
+
3428
+
3429
+
3430
+
3431
+
3432
+
3433
+
3434
+
3435
+
3436
+
3437
+
3438
+
3439
+
3440
+
3441
+
3442
+
3443
+
3444
+
3445
+
3446
+
3447
+
3448
+
3449
+
3450
+
3451
+
3452
+
3453
+
3454
+
3455
+
3456
+
3457
+
3458
+
3459
+
3460
+
3461
+
3462
+
3463
+
3464
+
3465
+
3466
+
3467
+
3468
+
3469
+
3470
+
3471
+
3472
+
3473
+
3474
+
3475
+
3476
+
3477
+
3478
+
3479
+ Training...: 24%|███████████████████████████████████████████▋ | 13999/59219 [2:01:48<6:05:37, 2.06it/s]4it/s]
3480
+
3481
+
3482
+
3483
+
3484
+
3485
+
3486
+
3487
+
3488
+
3489
+
3490
+
3491
+
3492
+
3493
+
3494
+
3495
+
3496
+
3497
+
3498
+
3499
+
3500
+
3501
+
3502
+
3503
+
3504
+
3505
+
3506
+
3507
+
3508
+
3509
+
3510
+
3511
+
3512
+
3513
+
3514
+
3515
+
3516
+
3517
+
3518
+
3519
+
3520
+
3521
+
3522
+
3523
+
3524
+
3525
+
3526
+
3527
+
3528
+
3529
+
3530
+
3531
+
3532
+
3533
+
3534
+
3535
+
3536
+
3537
+
3538
+
3539
+
3540
+
3541
+
3542
+
3543
+
3544
+
3545
+
3546
+
3547
+
3548
+
3549
+
3550
+
3551
+
3552
+
3553
+
3554
+
3555
+
3556
+
3557
+
3558
+
3559
+
3560
+
3561
+
3562
+
3563
+
3564
+
3565
+
3566
+
3567
+
3568
+
3569
+
3570
+
3571
+
3572
+
3573
+
3574
+
3575
+
3576
+
3577
+
3578
+
3579
+
3580
+
3581
+
3582
+
3583
+
3584
+
3585
+
3586
+
3587
+
3588
+
3589
+
3590
+
3591
+
3592
+
3593
+
3594
+
3595
+
3596
+
3597
+
3598
+
3599
+
3600
+
3601
+
3602
+
3603
+
3604
+
3605
+
3606
+
3607
+
3608
+
3609
+
3610
+
3611
+
3612
+
3613
+
3614
+
3615
+
3616
+
3617
+
3618
+
3619
+
3620
+
3621
+
3622
+
3623
+
3624
+
3625
+
3626
+
3627
+
3628
+
3629
+
3630
+
3631
+
3632
+
3633
+
3634
+
3635
+
3636
+
3637
+
3638
+
3639
+
3640
+
3641
+
3642
+
3643
+
3644
+
3645
+
3646
+
3647
+
3648
+
3649
+
3650
+
3651
+
3652
+
3653
+
3654
+
3655
+
3656
+
3657
+
3658
+
3659
+
3660
+
3661
+
3662
+
3663
+
3664
+
3665
+
3666
+
3667
+
3668
+
3669
+
3670
+
3671
+
3672
+
3673
+
3674
+
3675
+
3676
+
3677
+
3678
+
3679
+
3680
+
3681
+
3682
+
3683
+
3684
+
3685
+
3686
+
3687
+
3688
+
3689
+
3690
+
3691
+
3692
+
3693
+
3694
+
3695
+
3696
+
3697
+
3698
+
3699
+
3700
+
3701
+
3702
+
3703
+
3704
+
3705
+
3706
+
3707
+
3708
+
3709
+
3710
+
3711
+
3712
+
3713
+
3714
+ Training...: 25%|██████████████████████████████████████████████▊ | 14999/59219 [2:09:54<5:57:33, 2.06it/s]4it/s]
3715
+ Training...: 25%|██████████████████████████████████████████████▊ | 14999/59219 [2:10:11<5:57:33, 2.06it/s]4it/s]
3716
+
3717
+
3718
+
3719
+
3720
+
3721
+
3722
+
3723
+
3724
+
3725
+
3726
+
3727
+
3728
+
3729
+
3730
+
3731
+
3732
+
3733
+
3734
+
3735
+
3736
+
3737
+
3738
+
3739
+
3740
+
3741
+
3742
+
3743
+
3744
+
3745
+
3746
+
3747
+
3748
+
3749
+
3750
+
3751
+
3752
+
3753
+
3754
+
3755
+
3756
+
3757
+
3758
+
3759
+
3760
+
3761
+
3762
+
3763
+
3764
+
3765
+
3766
+
3767
+
3768
+
3769
+
3770
+
3771
+
3772
+
3773
+
3774
+
3775
+
3776
+
3777
+
3778
+
3779
+
3780
+
3781
+
3782
+
3783
+
3784
+
3785
+
3786
+
3787
+
3788
+
3789
+
3790
+
3791
+
3792
+
3793
+
3794
+
3795
+
3796
+
3797
+
3798
+
3799
+
3800
+
3801
+
3802
+
3803
+
3804
+
3805
+
3806
+
3807
+
3808
+
3809
+
3810
+
3811
+
3812
+
3813
+
3814
+
3815
+
3816
+
3817
+
3818
+
3819
+
3820
+
3821
+
3822
+
3823
+
3824
+
3825
+
3826
+
3827
+
3828
+
3829
+
3830
+
3831
+
3832
+
3833
+
3834
+
3835
+
3836
+
3837
+
3838
+
3839
+
3840
+
3841
+
3842
+
3843
+
3844
+
3845
+
3846
+
3847
+
3848
+
3849
+
3850
+
3851
+
3852
+
3853
+
3854
+
3855
+
3856
+
3857
+
3858
+
3859
+
3860
+
3861
+
3862
+
3863
+
3864
+
3865
+
3866
+
3867
+
3868
+
3869
+
3870
+
3871
+
3872
+
3873
+
3874
+
3875
+
3876
+
3877
+
3878
+
3879
+
3880
+
3881
+
3882
+
3883
+
3884
+
3885
+
3886
+
3887
+
3888
+
3889
+
3890
+
3891
+
3892
+
3893
+
3894
+
3895
+
3896
+
3897
+
3898
+
3899
+
3900
+
3901
+
3902
+
3903
+
3904
+
3905
+
3906
+
3907
+
3908
+
3909
+
3910
+
3911
+
3912
+
3913
+
3914
+
3915
+
3916
+
3917
+
3918
+
3919
+
3920
+
3921
+
3922
+
3923
+
3924
+
3925
+
3926
+
3927
+
3928
+
3929
+
3930
+
3931
+
3932
+
3933
+
3934
+
3935
+
3936
+
3937
+
3938
+
3939
+
3940
+
3941
+
3942
+
3943
+
3944
+
3945
+
3946
+
3947
+
3948
+
3949
+
3950
+
3951
+
3952
+
3953
+
3954
+
3955
+
3956
+
3957
+
3958
+
3959
+
3960
+
3961
+
3962
+
3963
+
3964
+
3965
+
3966
+
3967
+
3968
+
3969
+
3970
+
3971
+
3972
+
3973
+
3974
+
3975
+
3976
+
3977
+
3978
+
3979
+
3980
+
3981
+
3982
+
3983
+
3984
+
3985
+
3986
+
3987
+
3988
+
3989
+
3990
+
3991
+
3992
+
3993
+
3994
+
3995
+
3996
+
3997
+
3998
+
3999
+
4000
+
4001
+
4002
+
4003
+
4004
+
4005
+
4006
+
4007
+
4008
+
4009
+
4010
+
4011
+
4012
+
4013
+
4014
+
4015
+
4016
+
4017
+
4018
+
4019
+
4020
+
4021
+
4022
+
4023
+
4024
+
4025
+
4026
+
4027
+
4028
+
4029
+
4030
+
4031
+
4032
+
4033
+
4034
+
4035
+
4036
+ Training...: 27%|█████████████████████████████████████████████████▉ | 15999/59219 [2:21:02<5:49:15, 2.06it/s]4it/s]
4037
+
4038
+
4039
+
4040
+
4041
+
4042
+
4043
+
4044
+
4045
+
4046
+
4047
+
4048
+
4049
+
4050
+
4051
+
4052
+
4053
+
4054
+
4055
+
4056
+
4057
+
4058
+
4059
+
4060
+
4061
+
4062
+
4063
+
4064
+
4065
+
4066
+
4067
+
4068
+
4069
+
4070
+
4071
+
4072
+
4073
+
4074
+
4075
+
4076
+
4077
+
4078
+
4079
+
4080
+
4081
+
4082
+
4083
+
4084
+
4085
+
4086
+
4087
+
4088
+
4089
+
4090
+
4091
+
4092
+
4093
+
4094
+
4095
+
4096
+
4097
+
4098
+
4099
+
4100
+
4101
+
4102
+
4103
+
4104
+
4105
+
4106
+
4107
+
4108
+
4109
+
4110
+
4111
+
4112
+
4113
+
4114
+
4115
+
4116
+
4117
+
4118
+
4119
+
4120
+
4121
+
4122
+
4123
+
4124
+
4125
+
4126
+
4127
+
4128
+
4129
+
4130
+
4131
+
4132
+
4133
+
4134
+
4135
+
4136
+
4137
+
4138
+
4139
+
4140
+
4141
+
4142
+
4143
+
4144
+
4145
+
4146
+
4147
+
4148
+
4149
+
4150
+
4151
+
4152
+
4153
+
4154
+
4155
+
4156
+
4157
+
4158
+
4159
+
4160
+
4161
+
4162
+
4163
+
4164
+
4165
+
4166
+
4167
+
4168
+
4169
+
4170
+
4171
+
4172
+
4173
+
4174
+
4175
+
4176
+
4177
+
4178
+
4179
+
4180
+
4181
+
4182
+
4183
+
4184
+
4185
+
4186
+
4187
+
4188
+
4189
+
4190
+
4191
+
4192
+
4193
+
4194
+
4195
+
4196
+
4197
+
4198
+
4199
+
4200
+
4201
+
4202
+
4203
+
4204
+
4205
+
4206
+
4207
+
4208
+
4209
+
4210
+
4211
+
4212
+
4213
+
4214
+
4215
+
4216
+
4217
+
4218
+
4219
+
4220
+
4221
+
4222
+
4223
+
4224
+
4225
+
4226
+
4227
+
4228
+
4229
+
4230
+
4231
+
4232
+
4233
+
4234
+
4235
+
4236
+
4237
+
4238
+
4239
+
4240
+
4241
+
4242
+
4243
+
4244
+
4245
+
4246
+
4247
+
4248
+
4249
+
4250
+
4251
+
4252
+
4253
+
4254
+
4255
+
4256
+
4257
+
4258
+
4259
+
4260
+
4261
+
4262
+
4263
+
4264
+
4265
+
4266
+
4267
+
4268
+
4269
+
4270
+
4271
+ Training...: 29%|█████████████████████████████████████████████████████ | 16999/59219 [2:29:08<5:41:41, 2.06it/s]4it/s]
4272
+
4273
+
4274
+
4275
+
4276
+
4277
+
4278
+
4279
+
4280
+
4281
+
4282
+
4283
+
4284
+
4285
+
4286
+
4287
+
4288
+
4289
+
4290
+
4291
+
4292
+
4293
+
4294
+
4295
+
4296
+
4297
+
4298
+
4299
+
4300
+
4301
+
4302
+
4303
+
4304
+
4305
+
4306
+
4307
+
4308
+
4309
+
4310
+
4311
+
4312
+
4313
+
4314
+
4315
+
4316
+
4317
+
4318
+
4319
+
4320
+
4321
+
4322
+
4323
+
4324
+
4325
+
4326
+
4327
+
4328
+
4329
+
4330
+
4331
+
4332
+
4333
+
4334
+
4335
+
4336
+
4337
+
4338
+
4339
+
4340
+
4341
+
4342
+
4343
+
4344
+
4345
+
4346
+
4347
+
4348
+
4349
+
4350
+
4351
+
4352
+
4353
+
4354
+
4355
+
4356
+
4357
+
4358
+
4359
+
4360
+
4361
+
4362
+
4363
+
4364
+
4365
+
4366
+
4367
+
4368
+
4369
+
4370
+
4371
+
4372
+
4373
+
4374
+
4375
+
4376
+
4377
+
4378
+
4379
+
4380
+
4381
+
4382
+
4383
+
4384
+
4385
+
4386
+
4387
+
4388
+
4389
+
4390
+
4391
+
4392
+
4393
+
4394
+
4395
+
4396
+
4397
+
4398
+
4399
+
4400
+
4401
+
4402
+
4403
+
4404
+
4405
+
4406
+
4407
+
4408
+
4409
+
4410
+
4411
+
4412
+
4413
+
4414
+
4415
+
4416
+
4417
+
4418
+
4419
+
4420
+
4421
+
4422
+
4423
+
4424
+
4425
+
4426
+
4427
+
4428
+
4429
+
4430
+
4431
+
4432
+
4433
+
4434
+
4435
+
4436
+
4437
+
4438
+
4439
+
4440
+
4441
+
4442
+
4443
+
4444
+
4445
+
4446
+
4447
+
4448
+
4449
+
4450
+
4451
+
4452
+
4453
+
4454
+
4455
+
4456
+
4457
+
4458
+
4459
+
4460
+
4461
+
4462
+
4463
+
4464
+
4465
+
4466
+
4467
+
4468
+
4469
+
4470
+
4471
+
4472
+
4473
+
4474
+
4475
+
4476
+
4477
+
4478
+
4479
+
4480
+
4481
+
4482
+
4483
+
4484
+
4485
+
4486
+
4487
+
4488
+
4489
+
4490
+
4491
+
4492
+
4493
+
4494
+
4495
+
4496
+
4497
+
4498
+
4499
+
4500
+
4501
+
4502
+
4503
+
4504
+
4505
+
4506
+ Training...: 30%|████████████████████████████████████████████████████████▏ | 17999/59219 [2:37:14<5:33:12, 2.06it/s]4it/s]
4507
+
4508
+
4509
+
4510
+
4511
+
4512
+
4513
+
4514
+
4515
+
4516
+
4517
+
4518
+
4519
+
4520
+
4521
+
4522
+
4523
+
4524
+
4525
+
4526
+
4527
+
4528
+
4529
+
4530
+
4531
+
4532
+
4533
+
4534
+
4535
+
4536
+
4537
+
4538
+
4539
+
4540
+
4541
+
4542
+
4543
+
4544
+
4545
+
4546
+
4547
+
4548
+
4549
+
4550
+
4551
+
4552
+
4553
+
4554
+
4555
+
4556
+
4557
+
4558
+
4559
+
4560
+
4561
+
4562
+
4563
+
4564
+
4565
+
4566
+
4567
+
4568
+
4569
+
4570
+
4571
+
4572
+
4573
+
4574
+
4575
+
4576
+
4577
+
4578
+
4579
+
4580
+
4581
+
4582
+
4583
+
4584
+
4585
+
4586
+
4587
+
4588
+
4589
+
4590
+
4591
+
4592
+
4593
+
4594
+
4595
+
4596
+
4597
+
4598
+
4599
+
4600
+
4601
+
4602
+
4603
+
4604
+
4605
+
4606
+
4607
+
4608
+
4609
+
4610
+
4611
+
4612
+
4613
+
4614
+
4615
+
4616
+
4617
+
4618
+
4619
+
4620
+
4621
+
4622
+
4623
+
4624
+
4625
+
4626
+
4627
+
4628
+
4629
+
4630
+
4631
+
4632
+
4633
+
4634
+
4635
+
4636
+
4637
+
4638
+
4639
+
4640
+
4641
+
4642
+
4643
+
4644
+
4645
+
4646
+
4647
+
4648
+
4649
+
4650
+
4651
+
4652
+
4653
+
4654
+
4655
+
4656
+
4657
+
4658
+
4659
+
4660
+
4661
+
4662
+
4663
+
4664
+
4665
+
4666
+
4667
+
4668
+
4669
+
4670
+
4671
+
4672
+
4673
+
4674
+
4675
+
4676
+
4677
+
4678
+
4679
+
4680
+
4681
+
4682
+
4683
+
4684
+
4685
+
4686
+
4687
+
4688
+
4689
+
4690
+
4691
+
4692
+
4693
+
4694
+
4695
+
4696
+
4697
+
4698
+
4699
+
4700
+
4701
+
4702
+
4703
+
4704
+
4705
+
4706
+
4707
+
4708
+
4709
+
4710
+
4711
+
4712
+
4713
+
4714
+
4715
+
4716
+
4717
+
4718
+
4719
+
4720
+
4721
+
4722
+
4723
+
4724
+
4725
+
4726
+
4727
+
4728
+
4729
+
4730
+
4731
+
4732
+
4733
+
4734
+
4735
+
4736
+
4737
+
4738
+
4739
+
4740
+
4741
+ Training...: 32%|███████████████████████████████████████████████████████████▎ | 18999/59219 [2:45:21<5:25:06, 2.06it/s]4it/s]
4742
+
4743
+
4744
+
4745
+
4746
+
4747
+
4748
+
4749
+
4750
+
4751
+
4752
+
4753
+
4754
+
4755
+
4756
+
4757
+
4758
+
4759
+
4760
+
4761
+
4762
+
4763
+
4764
+
4765
+
4766
+
4767
+
4768
+
4769
+
4770
+
4771
+
4772
+
4773
+
4774
+
4775
+
4776
+
4777
+
4778
+
4779
+
4780
+
4781
+
4782
+
4783
+
4784
+
4785
+
4786
+
4787
+
4788
+
4789
+
4790
+
4791
+
4792
+
4793
+
4794
+
4795
+
4796
+
4797
+
4798
+
4799
+
4800
+
4801
+
4802
+
4803
+
4804
+
4805
+
4806
+
4807
+
4808
+
4809
+
4810
+
4811
+
4812
+
4813
+
4814
+
4815
+
4816
+
4817
+
4818
+
4819
+
4820
+
4821
+
4822
+
4823
+
4824
+
4825
+
4826
+
4827
+
4828
+
4829
+
4830
+
4831
+
4832
+
4833
+
4834
+
4835
+
4836
+
4837
+
4838
+
4839
+
4840
+
4841
+
4842
+
4843
+
4844
+
4845
+
4846
+
4847
+
4848
+
4849
+
4850
+
4851
+
4852
+
4853
+
4854
+
4855
+
4856
+
4857
+
4858
+
4859
+
4860
+
4861
+
4862
+
4863
+
4864
+
4865
+
4866
+
4867
+
4868
+
4869
+
4870
+
4871
+
4872
+
4873
+
4874
+
4875
+
4876
+
4877
+
4878
+
4879
+
4880
+
4881
+
4882
+
4883
+
4884
+
4885
+
4886
+
4887
+
4888
+
4889
+
4890
+
4891
+
4892
+
4893
+
4894
+
4895
+
4896
+
4897
+
4898
+
4899
+
4900
+
4901
+
4902
+
4903
+
4904
+
4905
+
4906
+
4907
+
4908
+
4909
+
4910
+
4911
+
4912
+
4913
+
4914
+
4915
+
4916
+
4917
+
4918
+
4919
+
4920
+
4921
+
4922
+
4923
+
4924
+
4925
+
4926
+
4927
+
4928
+
4929
+
4930
+
4931
+
4932
+
4933
+
4934
+
4935
+
4936
+
4937
+
4938
+
4939
+
4940
+
4941
+
4942
+
4943
+
4944
+
4945
+
4946
+
4947
+
4948
+
4949
+
4950
+
4951
+
4952
+
4953
+
4954
+
4955
+
4956
+
4957
+
4958
+
4959
+
4960
+
4961
+
4962
+
4963
+
4964
+
4965
+
4966
+
4967
+
4968
+
4969
+
4970
+
4971
+
4972
+
4973
+
4974
+
4975
+
4976
+
4977
+ Training...: 34%|██████████████████████████████████████████████████████████████▍ | 19999/59219 [2:53:44<5:16:56, 2.06it/s]4it/s]
4978
+ Evaluating ...: 0%| | 0/604 [00:00<?, ?it/s]
4979
+
4980
+
4981
+
4982
+
4983
+
4984
+
4985
+
4986
+
4987
+
4988
+
4989
+
4990
+
4991
+
4992
+
4993
+
4994
+
4995
+
4996
+
4997
+
4998
+
4999
+
5000
+
5001
+
5002
+
5003
+
5004
+
5005
+
5006
+
5007
+
5008
+
5009
+
5010
+
5011
+
5012
+
5013
+
5014
+
5015
+
5016
+
5017
+
5018
+
5019
+
5020
+
5021
+
5022
+
5023
+
5024
+
5025
+
5026
+
5027
+
5028
+
5029
+
5030
+
5031
+
5032
+
5033
+
5034
+
5035
+
5036
+
5037
+
5038
+
5039
+
5040
+
5041
+
5042
+
5043
+
5044
+
5045
+
5046
+
5047
+
5048
+
5049
+
5050
+
5051
+
5052
+
5053
+
5054
+
5055
+
5056
+
5057
+
5058
+
5059
+
5060
+
5061
+
5062
+
5063
+
5064
+
5065
+
5066
+
5067
+
5068
+
5069
+
5070
+
5071
+
5072
+
5073
+
5074
+
5075
+
5076
+
5077
+
5078
+
5079
+
5080
+
5081
+
5082
+
5083
+
5084
+
5085
+
5086
+
5087
+
5088
+
5089
+
5090
+
5091
+
5092
+
5093
+
5094
+
5095
+
5096
+
5097
+
5098
+
5099
+
5100
+
5101
+
5102
+
5103
+
5104
+
5105
+
5106
+
5107
+
5108
+
5109
+
5110
+
5111
+
5112
+
5113
+
5114
+
5115
+
5116
+
5117
+
5118
+
5119
+
5120
+
5121
+
5122
+
5123
+
5124
+
5125
+
5126
+
5127
+
5128
+
5129
+
5130
+
5131
+
5132
+
5133
+
5134
+
5135
+
5136
+
5137
+
5138
+
5139
+
5140
+
5141
+
5142
+
5143
+
5144
+
5145
+
5146
+
5147
+
5148
+
5149
+
5150
+
5151
+
5152
+
5153
+
5154
+
5155
+
5156
+
5157
+
5158
+
5159
+
5160
+
5161
+
5162
+
5163
+
5164
+
5165
+
5166
+
5167
+
5168
+
5169
+
5170
+
5171
+
5172
+
5173
+
5174
+
5175
+
5176
+
5177
+
5178
+
5179
+
5180
+
5181
+
5182
+
5183
+
5184
+
5185
+
5186
+
5187
+
5188
+
5189
+
5190
+
5191
+
5192
+
5193
+
5194
+
5195
+
5196
+
5197
+
5198
+
5199
+
5200
+
5201
+
5202
+
5203
+
5204
+
5205
+
5206
+
5207
+
5208
+
5209
+
5210
+
5211
+
5212
+
5213
+
5214
+
5215
+
5216
+
5217
+
5218
+
5219
+
5220
+
5221
+
5222
+
5223
+
5224
+
5225
+
5226
+
5227
+
5228
+
5229
+
5230
+
5231
+
5232
+
5233
+
5234
+
5235
+
5236
+
5237
+
5238
+
5239
+
5240
+
5241
+
5242
+
5243
+
5244
+
5245
+
5246
+
5247
+
5248
+
5249
+
5250
+
5251
+
5252
+
5253
+
5254
+
5255
+
5256
+
5257
+
5258
+
5259
+
5260
+
5261
+
5262
+
5263
+
5264
+
5265
+
5266
+
5267
+
5268
+
5269
+
5270
+
5271
+
5272
+
5273
+
5274
+
5275
+
5276
+
5277
+
5278
+
5279
+
5280
+
5281
+
5282
+
5283
+
5284
+
5285
+
5286
+
5287
+
5288
+
5289
+
5290
+
5291
+
5292
+
5293
+
5294
+
5295
+
5296
+
5297
+
5298
+
5299
+
5300
+ Training...: 35%|█████████████████████████████████████████████████████████████████▋ | 21012/59219 [3:04:52<4:16:34, 2.48it/s]4it/s]
5301
+
5302
+
5303
+
5304
+
5305
+
5306
+
5307
+
5308
+
5309
+
5310
+
5311
+
5312
+
5313
+
5314
+
5315
+
5316
+
5317
+
5318
+
5319
+
5320
+
5321
+
5322
+
5323
+
5324
+
5325
+
5326
+
5327
+
5328
+
5329
+
5330
+
5331
+
5332
+
5333
+
5334
+
5335
+
5336
+
5337
+
5338
+
5339
+
5340
+
5341
+
5342
+
5343
+
5344
+
5345
+
5346
+
5347
+
5348
+
5349
+
5350
+
5351
+
5352
+
5353
+
5354
+
5355
+
5356
+
5357
+
5358
+
5359
+
5360
+
5361
+
5362
+
5363
+
5364
+
5365
+
5366
+
5367
+
5368
+
5369
+
5370
+
5371
+
5372
+
5373
+
5374
+
5375
+
5376
+
5377
+
5378
+
5379
+
5380
+
5381
+
5382
+
5383
+
5384
+
5385
+
5386
+
5387
+
5388
+
5389
+
5390
+
5391
+
5392
+
5393
+
5394
+
5395
+
5396
+
5397
+
5398
+
5399
+
5400
+
5401
+
5402
+
5403
+
5404
+
5405
+
5406
+
5407
+
5408
+
5409
+
5410
+
5411
+
5412
+
5413
+
5414
+
5415
+
5416
+
5417
+
5418
+
5419
+
5420
+
5421
+
5422
+
5423
+
5424
+
5425
+
5426
+
5427
+
5428
+
5429
+
5430
+
5431
+
5432
+
5433
+
5434
+
5435
+
5436
+
5437
+
5438
+
5439
+
5440
+
5441
+
5442
+
5443
+
5444
+
5445
+
5446
+
5447
+
5448
+
5449
+
5450
+
5451
+
5452
+
5453
+
5454
+
5455
+
5456
+
5457
+
5458
+
5459
+
5460
+
5461
+
5462
+
5463
+
5464
+
5465
+
5466
+
5467
+
5468
+
5469
+
5470
+
5471
+
5472
+
5473
+
5474
+
5475
+
5476
+
5477
+
5478
+
5479
+
5480
+
5481
+
5482
+
5483
+
5484
+
5485
+
5486
+
5487
+
5488
+
5489
+
5490
+
5491
+
5492
+
5493
+
5494
+
5495
+
5496
+
5497
+
5498
+
5499
+
5500
+
5501
+
5502
+
5503
+
5504
+
5505
+
5506
+
5507
+
5508
+
5509
+
5510
+
5511
+
5512
+
5513
+
5514
+
5515
+
5516
+
5517
+
5518
+
5519
+
5520
+
5521
+
5522
+
5523
+
5524
+
5525
+
5526
+
5527
+
5528
+
5529
+
5530
+
5531
+
5532
+
5533
+
5534
+
5535
+ Training...: 37%|████████████████████████████████████████████████████████████████████▋ | 21999/59219 [3:12:41<5:01:06, 2.06it/s]4it/s]
5536
+
5537
+
5538
+
5539
+
5540
+
5541
+
5542
+
5543
+
5544
+
5545
+
5546
+
5547
+
5548
+
5549
+
5550
+
5551
+
5552
+
5553
+
5554
+
5555
+
5556
+
5557
+
5558
+
5559
+
5560
+
5561
+
5562
+
5563
+
5564
+
5565
+
5566
+
5567
+
5568
+
5569
+
5570
+
5571
+
5572
+
5573
+
5574
+
5575
+
5576
+
5577
+
5578
+
5579
+
5580
+
5581
+
5582
+
5583
+
5584
+
5585
+
5586
+
5587
+
5588
+
5589
+
5590
+
5591
+
5592
+
5593
+
5594
+
5595
+
5596
+
5597
+
5598
+
5599
+
5600
+
5601
+
5602
+
5603
+
5604
+
5605
+
5606
+
5607
+
5608
+
5609
+
5610
+
5611
+
5612
+
5613
+
5614
+
5615
+
5616
+
5617
+
5618
+
5619
+
5620
+
5621
+
5622
+
5623
+
5624
+
5625
+
5626
+
5627
+
5628
+
5629
+
5630
+
5631
+
5632
+
5633
+
5634
+
5635
+
5636
+
5637
+
5638
+
5639
+
5640
+
5641
+
5642
+
5643
+
5644
+
5645
+
5646
+
5647
+
5648
+
5649
+
5650
+
5651
+
5652
+
5653
+
5654
+
5655
+
5656
+
5657
+
5658
+
5659
+
5660
+
5661
+
5662
+
5663
+
5664
+
5665
+
5666
+
5667
+
5668
+
5669
+
5670
+
5671
+
5672
+
5673
+
5674
+
5675
+
5676
+
5677
+
5678
+
5679
+
5680
+
5681
+
5682
+
5683
+
5684
+
5685
+
5686
+
5687
+
5688
+
5689
+
5690
+
5691
+
5692
+
5693
+
5694
+
5695
+
5696
+
5697
+
5698
+
5699
+
5700
+
5701
+
5702
+
5703
+
5704
+
5705
+
5706
+
5707
+
5708
+
5709
+
5710
+
5711
+
5712
+
5713
+
5714
+
5715
+
5716
+
5717
+
5718
+
5719
+
5720
+
5721
+
5722
+
5723
+
5724
+
5725
+
5726
+
5727
+
5728
+
5729
+
5730
+
5731
+
5732
+
5733
+
5734
+
5735
+
5736
+
5737
+
5738
+
5739
+
5740
+
5741
+
5742
+
5743
+
5744
+
5745
+
5746
+
5747
+
5748
+
5749
+
5750
+
5751
+
5752
+
5753
+
5754
+
5755
+
5756
+
5757
+
5758
+
5759
+
5760
+
5761
+
5762
+
5763
+
5764
+
5765
+
5766
+
5767
+
5768
+
5769
+
5770
+
5771
+
5772
+ Training...: 39%|███████████████████████████████████████████████████████████████████████▍ | 23004/59219 [3:21:04<18:16:08, 1.82s/it]4it/s]
5773
+
5774
+
5775
+
5776
+
5777
+
5778
+
5779
+
5780
+
5781
+
5782
+
5783
+
5784
+
5785
+
5786
+
5787
+
5788
+
5789
+
5790
+
5791
+
5792
+
5793
+
5794
+
5795
+
5796
+
5797
+
5798
+
5799
+
5800
+
5801
+
5802
+
5803
+
5804
+
5805
+
5806
+
5807
+
5808
+
5809
+
5810
+
5811
+
5812
+
5813
+
5814
+
5815
+
5816
+
5817
+
5818
+
5819
+
5820
+
5821
+
5822
+
5823
+
5824
+
5825
+
5826
+
5827
+
5828
+
5829
+
5830
+
5831
+
5832
+
5833
+
5834
+
5835
+
5836
+
5837
+
5838
+
5839
+
5840
+
5841
+
5842
+
5843
+
5844
+
5845
+
5846
+
5847
+
5848
+
5849
+
5850
+
5851
+
5852
+
5853
+
5854
+
5855
+
5856
+
5857
+
5858
+
5859
+
5860
+
5861
+
5862
+
5863
+
5864
+
5865
+
5866
+
5867
+
5868
+
5869
+
5870
+
5871
+
5872
+
5873
+
5874
+
5875
+
5876
+
5877
+
5878
+
5879
+
5880
+
5881
+
5882
+
5883
+
5884
+
5885
+
5886
+
5887
+
5888
+
5889
+
5890
+
5891
+
5892
+
5893
+
5894
+
5895
+
5896
+
5897
+
5898
+
5899
+
5900
+
5901
+
5902
+
5903
+
5904
+
5905
+
5906
+
5907
+
5908
+
5909
+
5910
+
5911
+
5912
+
5913
+
5914
+
5915
+
5916
+
5917
+
5918
+
5919
+
5920
+
5921
+
5922
+
5923
+
5924
+
5925
+
5926
+
5927
+
5928
+
5929
+
5930
+
5931
+
5932
+
5933
+
5934
+
5935
+
5936
+
5937
+
5938
+
5939
+
5940
+
5941
+
5942
+
5943
+
5944
+
5945
+
5946
+
5947
+
5948
+
5949
+
5950
+
5951
+
5952
+
5953
+
5954
+
5955
+
5956
+
5957
+
5958
+
5959
+
5960
+
5961
+
5962
+
5963
+
5964
+
5965
+
5966
+
5967
+
5968
+
5969
+
5970
+
5971
+
5972
+
5973
+
5974
+
5975
+
5976
+
5977
+
5978
+
5979
+
5980
+
5981
+
5982
+
5983
+
5984
+
5985
+
5986
+
5987
+
5988
+
5989
+
5990
+
5991
+
5992
+
5993
+
5994
+
5995
+
5996
+
5997
+
5998
+
5999
+
6000
+
6001
+
6002
+
6003
+
6004
+
6005
+
6006
+
6007
+ Training...: 41%|██████████████████████████████████████████████████████████████████████████▉ | 23999/59219 [3:28:53<4:44:37, 2.06it/s]4it/s]
6008
+
6009
+
6010
+
6011
+
6012
+
6013
+
6014
+
6015
+
6016
+
6017
+
6018
+
6019
+
6020
+
6021
+
6022
+
6023
+
6024
+
6025
+
6026
+
6027
+
6028
+
6029
+
6030
+
6031
+
6032
+
6033
+
6034
+
6035
+
6036
+
6037
+
6038
+
6039
+
6040
+
6041
+
6042
+
6043
+
6044
+
6045
+
6046
+
6047
+
6048
+
6049
+
6050
+
6051
+
6052
+
6053
+
6054
+
6055
+
6056
+
6057
+
6058
+
6059
+
6060
+
6061
+
6062
+
6063
+
6064
+
6065
+
6066
+
6067
+
6068
+
6069
+
6070
+
6071
+
6072
+
6073
+
6074
+
6075
+
6076
+
6077
+
6078
+
6079
+
6080
+
6081
+
6082
+
6083
+
6084
+
6085
+
6086
+
6087
+
6088
+
6089
+
6090
+
6091
+
6092
+
6093
+
6094
+
6095
+
6096
+
6097
+
6098
+
6099
+
6100
+
6101
+
6102
+
6103
+
6104
+
6105
+
6106
+
6107
+
6108
+
6109
+
6110
+
6111
+
6112
+
6113
+
6114
+
6115
+
6116
+
6117
+
6118
+
6119
+
6120
+
6121
+
6122
+
6123
+
6124
+
6125
+
6126
+
6127
+
6128
+
6129
+
6130
+
6131
+
6132
+
6133
+
6134
+
6135
+
6136
+
6137
+
6138
+
6139
+
6140
+
6141
+
6142
+
6143
+
6144
+
6145
+
6146
+
6147
+
6148
+
6149
+
6150
+
6151
+
6152
+
6153
+
6154
+
6155
+
6156
+
6157
+
6158
+
6159
+
6160
+
6161
+
6162
+
6163
+
6164
+
6165
+
6166
+
6167
+
6168
+
6169
+
6170
+
6171
+
6172
+
6173
+
6174
+
6175
+
6176
+
6177
+
6178
+
6179
+
6180
+
6181
+
6182
+
6183
+
6184
+
6185
+
6186
+
6187
+
6188
+
6189
+
6190
+
6191
+
6192
+
6193
+
6194
+
6195
+
6196
+
6197
+
6198
+
6199
+
6200
+
6201
+
6202
+
6203
+
6204
+
6205
+
6206
+
6207
+
6208
+
6209
+
6210
+
6211
+
6212
+
6213
+
6214
+
6215
+
6216
+
6217
+
6218
+
6219
+
6220
+
6221
+
6222
+
6223
+
6224
+
6225
+
6226
+
6227
+
6228
+
6229
+
6230
+
6231
+
6232
+
6233
+
6234
+
6235
+
6236
+
6237
+
6238
+
6239
+
6240
+
6241
+
6242
+
6243
+ Training...: 42%|██████████████████████████████████████████████████████████████████████████████ | 24999/59219 [3:36:59<4:36:53, 2.06it/s]4it/s]
6244
+ Training...: 42%|██████████████████████████████████████████████████████████████████████████████ | 24999/59219 [3:37:16<4:36:53, 2.06it/s]4it/s]
6245
+
6246
+
6247
+
6248
+
6249
+
6250
+
6251
+
6252
+
6253
+
6254
+
6255
+
6256
+
6257
+
6258
+
6259
+
6260
+
6261
+
6262
+
6263
+
6264
+
6265
+
6266
+
6267
+
6268
+
6269
+
6270
+
6271
+
6272
+
6273
+
6274
+
6275
+
6276
+
6277
+
6278
+
6279
+
6280
+
6281
+
6282
+
6283
+
6284
+
6285
+
6286
+
6287
+
6288
+
6289
+
6290
+
6291
+
6292
+
6293
+
6294
+
6295
+
6296
+
6297
+
6298
+
6299
+
6300
+
6301
+
6302
+
6303
+
6304
+
6305
+
6306
+
6307
+
6308
+
6309
+
6310
+
6311
+
6312
+
6313
+
6314
+
6315
+
6316
+
6317
+
6318
+
6319
+
6320
+
6321
+
6322
+
6323
+
6324
+
6325
+
6326
+
6327
+
6328
+
6329
+
6330
+ tcmalloc: large alloc 1212121088 bytes == 0x391c76000 @ 0x7ff058e15680 0x7ff058e35bdd 0x7feea9b6b20d 0x7feea9b78069 0x7feea9b7956e 0x7feea9b74bd3 0x7feea9b751fe 0x5042c6 0x56bf09 0x56a0ba 0x5f6343 0x56cf2a 0x5f6166 0x56bddd 0x7feea9b6b410 0x7feea9b6c67a 0x7feea9b6e6f7 0x7feea9b782a3 0x7feea9b78e87 0x7feea9b78e87 0x7feea9b78e87 0x7feea9b78e87 0x7feea9b78e87 0x7feea9b74bd3 0x7feea9b751fe 0x5042c6 0x56bf09 0x56a0ba 0x5f6343 0x56cf2a 0x56a0ba
6331
+ tcmalloc: large alloc 3161391104 bytes == 0x447e68000 @ 0x7ff058e15680 0x7ff058e35bdd 0x7feea9b6b20d 0x7feea9b79340 0x7feea9b78e87 0x7feea9b78e87 0x7feea9b78e87 0x7feea9b78e87 0x7feea9b78e87 0x7feea9b78e87 0x7feea9b78e87 0x7feea9b74bd3 0x7feea9b751fe 0x5042c6 0x56bf09 0x56a0ba 0x5f6343 0x56cf2a 0x56a0ba 0x5f6343 0x56cf2a 0x5f6166 0x570e46 0x56a0ba 0x5f6343 0x56cf2a 0x56a0ba 0x68d5b7 0x67cd01 0x67cd7f 0x67ce21
6332
+ tcmalloc: large alloc 2886115328 bytes == 0x391c76000 @ 0x7ff058e15680 0x7ff058e36824 0x5f9f21 0x7feea9b74c6f 0x7feea9b751fe 0x5042c6 0x56bf09 0x56a0ba 0x5f6343 0x56cf2a 0x56a0ba 0x5f6343 0x56cf2a 0x5f6166 0x570e46 0x56a0ba 0x5f6343 0x56cf2a 0x56a0ba 0x68d5b7 0x67cd01 0x67cd7f 0x67ce21 0x67ef47 0x6b7242 0x6b75cd 0x7ff058c2c0b3 0x5fb18e
6333
+ [19:43:44] - INFO - absl - Saved checkpoint at ./checkpoint_125000
6334
+ [19:43:44] - INFO - absl - Removing checkpoint at ./checkpoint_100000
6335
+ Adding files tracked by Git LFS: ['checkpoint_125000']. This may take a bit of time if the files are large.
wandb/run-20211108_160143-33jzyy4p/files/requirements.txt ADDED
@@ -0,0 +1,101 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ absl-py==0.15.0
2
+ aiohttp==3.8.0
3
+ aiosignal==1.2.0
4
+ astunparse==1.6.3
5
+ async-timeout==4.0.0
6
+ attrs==21.2.0
7
+ cachetools==4.2.4
8
+ certifi==2021.10.8
9
+ charset-normalizer==2.0.7
10
+ chex==0.0.8
11
+ click==8.0.3
12
+ configparser==5.1.0
13
+ cycler==0.11.0
14
+ datasets==1.15.1
15
+ dill==0.3.4
16
+ dm-tree==0.1.6
17
+ docker-pycreds==0.4.0
18
+ filelock==3.3.2
19
+ flatbuffers==2.0
20
+ flax==0.3.6
21
+ frozenlist==1.2.0
22
+ fsspec==2021.11.0
23
+ gast==0.4.0
24
+ gitdb==4.0.9
25
+ gitpython==3.1.24
26
+ google-auth-oauthlib==0.4.6
27
+ google-auth==2.3.3
28
+ google-pasta==0.2.0
29
+ grpcio==1.41.1
30
+ h5py==3.5.0
31
+ huggingface-hub==0.1.1
32
+ idna==3.3
33
+ jax==0.2.24
34
+ jaxlib==0.1.73
35
+ joblib==1.1.0
36
+ kaggle==1.5.12
37
+ keras-preprocessing==1.1.2
38
+ keras==2.7.0
39
+ kiwisolver==1.3.2
40
+ libclang==12.0.0
41
+ libtpu-nightly==0.1.dev20211018
42
+ markdown==3.3.4
43
+ matplotlib==3.4.3
44
+ msgpack==1.0.2
45
+ multidict==5.2.0
46
+ multiprocess==0.70.12.2
47
+ numpy==1.21.4
48
+ oauthlib==3.1.1
49
+ opt-einsum==3.3.0
50
+ optax==0.0.9
51
+ packaging==21.2
52
+ pandas==1.3.4
53
+ pathtools==0.1.2
54
+ pillow==8.4.0
55
+ pip==21.3.1
56
+ pkg-resources==0.0.0
57
+ promise==2.3
58
+ protobuf==3.19.1
59
+ psutil==5.8.0
60
+ pyarrow==6.0.0
61
+ pyasn1-modules==0.2.8
62
+ pyasn1==0.4.8
63
+ pyparsing==2.4.7
64
+ python-dateutil==2.8.2
65
+ python-slugify==5.0.2
66
+ pytz==2021.3
67
+ pyyaml==6.0
68
+ regex==2021.11.2
69
+ requests-oauthlib==1.3.0
70
+ requests==2.26.0
71
+ rsa==4.7.2
72
+ sacremoses==0.0.46
73
+ scipy==1.7.1
74
+ sentry-sdk==1.4.3
75
+ setuptools==44.0.0
76
+ shortuuid==1.0.1
77
+ six==1.16.0
78
+ smmap==5.0.0
79
+ subprocess32==3.5.4
80
+ tensorboard-data-server==0.6.1
81
+ tensorboard-plugin-wit==1.8.0
82
+ tensorboard==2.7.0
83
+ tensorflow-estimator==2.7.0
84
+ tensorflow-io-gcs-filesystem==0.21.0
85
+ tensorflow==2.7.0
86
+ termcolor==1.1.0
87
+ text-unidecode==1.3
88
+ tokenizers==0.10.3
89
+ toolz==0.11.1
90
+ torch==1.10.0
91
+ tqdm==4.62.3
92
+ transformers==4.13.0.dev0
93
+ typing-extensions==3.10.0.2
94
+ urllib3==1.26.7
95
+ wandb==0.12.6
96
+ werkzeug==2.0.2
97
+ wheel==0.37.0
98
+ wrapt==1.13.3
99
+ xxhash==2.0.2
100
+ yarl==1.7.2
101
+ yaspin==2.1.0
wandb/run-20211108_160143-33jzyy4p/files/wandb-metadata.json ADDED
@@ -0,0 +1,69 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29",
3
+ "python": "3.8.10",
4
+ "heartbeatAt": "2021-11-08T16:01:46.046817",
5
+ "startedAt": "2021-11-08T16:01:43.059885",
6
+ "docker": null,
7
+ "cpu_count": 96,
8
+ "cuda": null,
9
+ "args": [
10
+ "--model_name_or_path",
11
+ "muril-bigbird",
12
+ "--tokenizer_name",
13
+ "muril-bigbird",
14
+ "--dtype",
15
+ "bfloat16",
16
+ "--pretokenized",
17
+ "--resume_from_checkpoint",
18
+ "checkpoint_100000",
19
+ "--train_file",
20
+ "/dev/shm/data-files",
21
+ "--validation_file",
22
+ ".",
23
+ "--max_seq_length",
24
+ "4096",
25
+ "--pad_to_max_length",
26
+ "--output_dir",
27
+ "./",
28
+ "--overwrite_output_dir",
29
+ "--do_train",
30
+ "--do_eval",
31
+ "--eval_steps",
32
+ "5000",
33
+ "--per_device_train_batch_size",
34
+ "1",
35
+ "--per_device_eval_batch_size",
36
+ "2",
37
+ "--learning_rate",
38
+ "5e-5",
39
+ "--weight_decay",
40
+ "0.01",
41
+ "--num_train_epochs",
42
+ "3",
43
+ "--warmup_steps",
44
+ "0",
45
+ "--logging_steps",
46
+ "1000",
47
+ "--save_steps",
48
+ "25000",
49
+ "--save_total_limit",
50
+ "1",
51
+ "--seed",
52
+ "18",
53
+ "--push_to_hub",
54
+ "--run_name",
55
+ "en-ta-hi-te-from-150k"
56
+ ],
57
+ "state": "running",
58
+ "program": "run_muril_bb_flax_mlm.py",
59
+ "codePath": "run_muril_bb_flax_mlm.py",
60
+ "git": {
61
+ "remote": "https://huggingface.co/nbroad/muril-bigbird-base-1mil",
62
+ "commit": "0c3c6800eaff5ab9efc1ff794cec7b611b79744f"
63
+ },
64
+ "email": "nicholas@nmbroad.com",
65
+ "root": "/home/nicholasbroad/muril-bigbird-base-1mil",
66
+ "host": "t1v-n-ba7aa1a5-w-0",
67
+ "username": "nicholasbroad",
68
+ "executable": "/home/nicholasbroad/venv/bin/python"
69
+ }
wandb/run-20211108_160143-33jzyy4p/files/wandb-summary.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"training_step": 125000, "learning_rate": 1.4819847820035648e-05, "train_loss": 0.8347105979919434, "_runtime": 13308, "_timestamp": 1636400611, "_step": 29, "eval_step": 125000, "eval_accuracy": 0.7502975463867188, "eval_loss": 1.2368080615997314}
wandb/run-20211108_160143-33jzyy4p/logs/debug-internal.log ADDED
The diff for this file is too large to render. See raw diff
 
wandb/run-20211108_160143-33jzyy4p/logs/debug.log ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2021-11-08 16:01:43,061 INFO MainThread:10728 [wandb_setup.py:_flush():71] setting env: {}
2
+ 2021-11-08 16:01:43,061 INFO MainThread:10728 [wandb_setup.py:_flush():71] setting login settings: {}
3
+ 2021-11-08 16:01:43,061 INFO MainThread:10728 [wandb_init.py:_log_setup():357] Logging user logs to /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_160143-33jzyy4p/logs/debug.log
4
+ 2021-11-08 16:01:43,061 INFO MainThread:10728 [wandb_init.py:_log_setup():358] Logging internal logs to /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_160143-33jzyy4p/logs/debug-internal.log
5
+ 2021-11-08 16:01:43,061 INFO MainThread:10728 [wandb_init.py:init():390] calling init triggers
6
+ 2021-11-08 16:01:43,061 INFO MainThread:10728 [wandb_init.py:init():395] wandb.init called with sweep_config: {}
7
+ config: {}
8
+ 2021-11-08 16:01:43,062 INFO MainThread:10728 [wandb_init.py:init():435] starting backend
9
+ 2021-11-08 16:01:43,062 INFO MainThread:10728 [backend.py:_multiprocessing_setup():95] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
10
+ 2021-11-08 16:01:43,098 INFO MainThread:10728 [backend.py:ensure_launched():199] starting backend process...
11
+ 2021-11-08 16:01:43,132 INFO MainThread:10728 [backend.py:ensure_launched():204] started backend process with pid: 12038
12
+ 2021-11-08 16:01:43,134 INFO MainThread:10728 [wandb_init.py:init():444] backend started and connected
13
+ 2021-11-08 16:01:43,137 INFO MainThread:10728 [wandb_init.py:init():503] updated telemetry
14
+ 2021-11-08 16:01:43,213 INFO MainThread:10728 [wandb_init.py:init():533] communicating current version
15
+ 2021-11-08 16:01:43,736 INFO MainThread:10728 [wandb_init.py:init():538] got version response
16
+ 2021-11-08 16:01:43,737 INFO MainThread:10728 [wandb_init.py:init():548] communicating run to backend with 30 second timeout
17
+ 2021-11-08 16:01:43,913 INFO MainThread:10728 [wandb_init.py:init():576] starting run threads in backend
18
+ 2021-11-08 16:01:46,085 INFO MainThread:10728 [wandb_run.py:_console_start():1693] atexit reg
19
+ 2021-11-08 16:01:46,085 INFO MainThread:10728 [wandb_run.py:_redirect():1567] redirect: SettingsConsole.REDIRECT
20
+ 2021-11-08 16:01:46,086 INFO MainThread:10728 [wandb_run.py:_redirect():1572] Redirecting console.
21
+ 2021-11-08 16:01:46,088 INFO MainThread:10728 [wandb_run.py:_redirect():1628] Redirects installed.
22
+ 2021-11-08 16:01:46,088 INFO MainThread:10728 [wandb_init.py:init():603] run started, returning control to user process
23
+ 2021-11-08 16:01:46,093 INFO MainThread:10728 [wandb_run.py:_config_callback():911] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': True, 'do_eval': True, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 2, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 5e-05, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 3.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 0, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Nov08_16-00-48_t1v-n-ba7aa1a5-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 1000, 'logging_nan_inf_filter': True, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 25000, 'save_total_limit': 1, 'save_on_each_node': False, 'no_cuda': False, 'seed': 18, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'xpu_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 5000, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': 'en-ta-hi-te-from-150k', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': 'checkpoint_100000', 'hub_model_id': None, 'hub_strategy': 'HubStrategy.EVERY_SAVE', 'hub_token': None, 'gradient_checkpointing': False, 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': '', '_n_gpu': 0, '__cached__setup_devices': 'cpu'}
24
+ 2021-11-08 16:01:46,095 INFO MainThread:10728 [wandb_run.py:_config_callback():911] config_cb None None {'model_name_or_path': 'muril-bigbird', 'model_type': None, 'config_name': None, 'tokenizer_name': 'muril-bigbird', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'bfloat16'}
25
+ 2021-11-08 16:01:46,097 INFO MainThread:10728 [wandb_run.py:_config_callback():911] config_cb None None {'dataset_name': None, 'pretokenized': True, 'streaming': False, 'dataset_config_name': None, 'train_file': '/dev/shm/data-files', 'validation_file': '.', 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': None, 'mlm_probability': 0.15, 'pad_to_max_length': True, 'line_by_line': False}
wandb/run-20211108_160143-33jzyy4p/run-33jzyy4p.wandb ADDED
Binary file (2.43 MB). View file