Hugging Face's logo

nbroad
/
muril-bigbird-base

Fill-Mask
Transformers
JAX
TensorBoard
big_bird
Model card Files
xet
 Metrics Community
1
nbroad commited on
Commit
2968346
·
1 Parent(s): 5fca06e

Saving weights and logs of step 75000

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +1 -1
  2. 100k_flax_model.msgpack +3 -0
  3. 50k_steps_flax_model.msgpack +3 -0
  4. checkpoint_100000 → checkpoint_75000 +1 -1
  5. events.out.tfevents.1636318087.t1v-n-ba7aa1a5-w-0.20231.0.v2 +2 -2
  6. events.out.tfevents.1636357015.t1v-n-ba7aa1a5-w-0.29588.0.v2 +3 -0
  7. events.out.tfevents.1636358735.t1v-n-ba7aa1a5-w-0.3899.0.v2 +3 -0
  8. events.out.tfevents.1636358897.t1v-n-ba7aa1a5-w-0.5478.0.v2 +3 -0
  9. events.out.tfevents.1636358979.t1v-n-ba7aa1a5-w-0.6978.0.v2 +3 -0
  10. events.out.tfevents.1636359665.t1v-n-ba7aa1a5-w-0.3923.0.v2 +3 -0
  11. file_order +325 -0
  12. flax_model.msgpack +1 -1
  13. muril-bigbird/flax_model.msgpack +1 -1
  14. run_muril_bb_flax.sh +4 -3
  15. run_muril_bb_flax_mlm.py +1 -1
  16. wandb/debug-internal.log +1 -1
  17. wandb/debug.log +1 -1
  18. wandb/latest-run +1 -1
  19. wandb/run-20211107_204808-2ns3lo5n/files/config.yaml +6 -0
  20. wandb/run-20211107_204808-2ns3lo5n/files/output.log +5414 -0
  21. wandb/run-20211107_204808-2ns3lo5n/files/wandb-summary.json +1 -1
  22. wandb/run-20211107_204808-2ns3lo5n/logs/debug-internal.log +0 -0
  23. wandb/run-20211107_204808-2ns3lo5n/logs/debug.log +122 -0
  24. wandb/run-20211107_204808-2ns3lo5n/run-2ns3lo5n.wandb +0 -0
  25. wandb/run-20211108_073656-hyqam6zi/files/config.yaml +347 -0
  26. wandb/run-20211108_073656-hyqam6zi/files/output.log +15 -0
  27. wandb/run-20211108_073656-hyqam6zi/files/requirements.txt +101 -0
  28. wandb/run-20211108_073656-hyqam6zi/files/wandb-metadata.json +69 -0
  29. wandb/run-20211108_073656-hyqam6zi/files/wandb-summary.json +1 -0
  30. wandb/run-20211108_073656-hyqam6zi/logs/debug-internal.log +145 -0
  31. wandb/run-20211108_073656-hyqam6zi/logs/debug.log +129 -0
  32. wandb/run-20211108_073656-hyqam6zi/run-hyqam6zi.wandb +0 -0
  33. wandb/run-20211108_080537-1gq01aek/files/config.yaml +347 -0
  34. wandb/run-20211108_080537-1gq01aek/files/output.log +16 -0
  35. wandb/run-20211108_080537-1gq01aek/files/requirements.txt +101 -0
  36. wandb/run-20211108_080537-1gq01aek/files/wandb-metadata.json +69 -0
  37. wandb/run-20211108_080537-1gq01aek/files/wandb-summary.json +1 -0
  38. wandb/run-20211108_080537-1gq01aek/logs/debug-internal.log +159 -0
  39. wandb/run-20211108_080537-1gq01aek/logs/debug.log +137 -0
  40. wandb/run-20211108_080537-1gq01aek/run-1gq01aek.wandb +0 -0
  41. wandb/run-20211108_080817-3dz8poa1/files/config.yaml +347 -0
  42. wandb/run-20211108_080817-3dz8poa1/files/output.log +120 -0
  43. wandb/run-20211108_080817-3dz8poa1/files/requirements.txt +101 -0
  44. wandb/run-20211108_080817-3dz8poa1/files/wandb-metadata.json +69 -0
  45. wandb/run-20211108_080817-3dz8poa1/files/wandb-summary.json +1 -0
  46. wandb/run-20211108_080817-3dz8poa1/logs/debug-internal.log +142 -0
  47. wandb/run-20211108_080817-3dz8poa1/logs/debug.log +129 -0
  48. wandb/run-20211108_080817-3dz8poa1/run-3dz8poa1.wandb +0 -0
  49. wandb/run-20211108_080940-r077t6tw/files/config.yaml +347 -0
  50. wandb/run-20211108_080940-r077t6tw/files/output.log +108 -0
.gitattributes CHANGED
@@ -25,4 +25,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
25
  *.zip filter=lfs diff=lfs merge=lfs -text
26
  *.zstandard filter=lfs diff=lfs merge=lfs -text
27
  *tfevents* filter=lfs diff=lfs merge=lfs -text
28
- checkpoint_100000 filter=lfs diff=lfs merge=lfs -text
 
25
  *.zip filter=lfs diff=lfs merge=lfs -text
26
  *.zstandard filter=lfs diff=lfs merge=lfs -text
27
  *tfevents* filter=lfs diff=lfs merge=lfs -text
28
+ checkpoint_75000 filter=lfs diff=lfs merge=lfs -text
100k_flax_model.msgpack ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9a27a51818882c80dc60c00a6758070d78bd85e45a66c91a5c31cf6f491516c7
3
+ size 962037499
50k_steps_flax_model.msgpack ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:92b4eb4b41a4fdd29fa296ca969236da8c1c0c923dab34bfff0448484f2a8228
3
+ size 962037499
checkpoint_100000 → checkpoint_75000 RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:56c3e577fcdc30358244e17911e0416068dbca5537ec3b8551631f3c4b90f426
3
  size 2886112612
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c386829fc369ca03d6970bef9037c52c9956540ef0a3585392813a01e37be848
3
  size 2886112612
events.out.tfevents.1636318087.t1v-n-ba7aa1a5-w-0.20231.0.v2 CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:c397e3f84b881b95bfb96cf45f8deb690133ae5a5da771e387c5ad225716277d
3
- size 7458821
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dbdb3cab3b1246125c212bd6a06cf039f52a25061a02bbfc94a2bf6f91ecbd55
3
+ size 10628868
events.out.tfevents.1636357015.t1v-n-ba7aa1a5-w-0.29588.0.v2 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4a4801ba99bdb8213d87b8d4f451b1c0149720e5c96f5efd5fd5752302afd7a4
3
+ size 40
events.out.tfevents.1636358735.t1v-n-ba7aa1a5-w-0.3899.0.v2 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:07930aa307734cd757b8ae37bcff122d28ce01678833a1835e60f45715d53ea1
3
+ size 40
events.out.tfevents.1636358897.t1v-n-ba7aa1a5-w-0.5478.0.v2 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:28340cce916b145b1c5caa26857d4ff0c75aa4dd29667cd2a7fe510949a01db6
3
+ size 40
events.out.tfevents.1636358979.t1v-n-ba7aa1a5-w-0.6978.0.v2 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6673d0330fb01bef206c1e839b76ade7ce1265b2ddf6e8f8f7cafd23136f88d8
3
+ size 40
events.out.tfevents.1636359665.t1v-n-ba7aa1a5-w-0.3923.0.v2 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:320e33fc26d13a5019c71f012f5d91e73064e4518fda8c450a307606d34d4b0f
3
+ size 3727500
file_order ADDED
@@ -0,0 +1,325 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ /dev/shm/data-files/telugu-data/9.parquet
2
+ /dev/shm/data-files/telugu-data/8.parquet
3
+ /dev/shm/data-files/telugu-data/7.parquet
4
+ /dev/shm/data-files/telugu-data/6.parquet
5
+ /dev/shm/data-files/telugu-data/5.parquet
6
+ /dev/shm/data-files/telugu-data/4.parquet
7
+ /dev/shm/data-files/telugu-data/3.parquet
8
+ /dev/shm/data-files/telugu-data/23.parquet
9
+ /dev/shm/data-files/telugu-data/22.parquet
10
+ /dev/shm/data-files/telugu-data/21.parquet
11
+ /dev/shm/data-files/telugu-data/20.parquet
12
+ /dev/shm/data-files/telugu-data/2.parquet
13
+ /dev/shm/data-files/telugu-data/19.parquet
14
+ /dev/shm/data-files/telugu-data/18.parquet
15
+ /dev/shm/data-files/telugu-data/17.parquet
16
+ /dev/shm/data-files/telugu-data/16.parquet
17
+ /dev/shm/data-files/telugu-data/15.parquet
18
+ /dev/shm/data-files/telugu-data/14.parquet
19
+ /dev/shm/data-files/telugu-data/13.parquet
20
+ /dev/shm/data-files/telugu-data/12.parquet
21
+ /dev/shm/data-files/telugu-data/11.parquet
22
+ /dev/shm/data-files/telugu-data/10.parquet
23
+ /dev/shm/data-files/telugu-data/1.parquet
24
+ /dev/shm/data-files/telugu-data/0.parquet
25
+ /dev/shm/data-files/tamil-data/9.parquet
26
+ /dev/shm/data-files/tamil-data/8.parquet
27
+ /dev/shm/data-files/tamil-data/7.parquet
28
+ /dev/shm/data-files/tamil-data/65.parquet
29
+ /dev/shm/data-files/tamil-data/64.parquet
30
+ /dev/shm/data-files/tamil-data/63.parquet
31
+ /dev/shm/data-files/tamil-data/62.parquet
32
+ /dev/shm/data-files/tamil-data/61.parquet
33
+ /dev/shm/data-files/tamil-data/60.parquet
34
+ /dev/shm/data-files/tamil-data/6.parquet
35
+ /dev/shm/data-files/tamil-data/59.parquet
36
+ /dev/shm/data-files/tamil-data/58.parquet
37
+ /dev/shm/data-files/tamil-data/57.parquet
38
+ /dev/shm/data-files/tamil-data/56.parquet
39
+ /dev/shm/data-files/tamil-data/55.parquet
40
+ /dev/shm/data-files/tamil-data/54.parquet
41
+ /dev/shm/data-files/tamil-data/53.parquet
42
+ /dev/shm/data-files/tamil-data/52.parquet
43
+ /dev/shm/data-files/tamil-data/51.parquet
44
+ /dev/shm/data-files/tamil-data/50.parquet
45
+ /dev/shm/data-files/tamil-data/5.parquet
46
+ /dev/shm/data-files/tamil-data/49.parquet
47
+ /dev/shm/data-files/tamil-data/48.parquet
48
+ /dev/shm/data-files/tamil-data/47.parquet
49
+ /dev/shm/data-files/tamil-data/46.parquet
50
+ /dev/shm/data-files/tamil-data/45.parquet
51
+ /dev/shm/data-files/tamil-data/44.parquet
52
+ /dev/shm/data-files/tamil-data/43.parquet
53
+ /dev/shm/data-files/tamil-data/42.parquet
54
+ /dev/shm/data-files/tamil-data/41.parquet
55
+ /dev/shm/data-files/tamil-data/40.parquet
56
+ /dev/shm/data-files/tamil-data/4.parquet
57
+ /dev/shm/data-files/tamil-data/39.parquet
58
+ /dev/shm/data-files/tamil-data/38.parquet
59
+ /dev/shm/data-files/tamil-data/37.parquet
60
+ /dev/shm/data-files/tamil-data/36.parquet
61
+ /dev/shm/data-files/tamil-data/35.parquet
62
+ /dev/shm/data-files/tamil-data/34.parquet
63
+ /dev/shm/data-files/tamil-data/33.parquet
64
+ /dev/shm/data-files/tamil-data/32.parquet
65
+ /dev/shm/data-files/tamil-data/31.parquet
66
+ /dev/shm/data-files/tamil-data/30.parquet
67
+ /dev/shm/data-files/tamil-data/3.parquet
68
+ /dev/shm/data-files/tamil-data/29.parquet
69
+ /dev/shm/data-files/tamil-data/28.parquet
70
+ /dev/shm/data-files/tamil-data/27.parquet
71
+ /dev/shm/data-files/tamil-data/26.parquet
72
+ /dev/shm/data-files/tamil-data/25.parquet
73
+ /dev/shm/data-files/tamil-data/24.parquet
74
+ /dev/shm/data-files/tamil-data/23.parquet
75
+ /dev/shm/data-files/tamil-data/22.parquet
76
+ /dev/shm/data-files/tamil-data/21.parquet
77
+ /dev/shm/data-files/tamil-data/20.parquet
78
+ /dev/shm/data-files/tamil-data/2.parquet
79
+ /dev/shm/data-files/tamil-data/19.parquet
80
+ /dev/shm/data-files/tamil-data/18.parquet
81
+ /dev/shm/data-files/tamil-data/17.parquet
82
+ /dev/shm/data-files/tamil-data/16.parquet
83
+ /dev/shm/data-files/tamil-data/15.parquet
84
+ /dev/shm/data-files/tamil-data/14.parquet
85
+ /dev/shm/data-files/tamil-data/13.parquet
86
+ /dev/shm/data-files/tamil-data/12.parquet
87
+ /dev/shm/data-files/tamil-data/11.parquet
88
+ /dev/shm/data-files/tamil-data/10.parquet
89
+ /dev/shm/data-files/tamil-data/1.parquet
90
+ /dev/shm/data-files/tamil-data/0.parquet
91
+ /dev/shm/data-files/hindi-data3/99.parquet
92
+ /dev/shm/data-files/hindi-data3/98.parquet
93
+ /dev/shm/data-files/hindi-data3/97.parquet
94
+ /dev/shm/data-files/hindi-data3/96.parquet
95
+ /dev/shm/data-files/hindi-data3/95.parquet
96
+ /dev/shm/data-files/hindi-data3/94.parquet
97
+ /dev/shm/data-files/hindi-data3/93.parquet
98
+ /dev/shm/data-files/hindi-data3/92.parquet
99
+ /dev/shm/data-files/hindi-data3/91.parquet
100
+ /dev/shm/data-files/hindi-data3/90.parquet
101
+ /dev/shm/data-files/hindi-data3/9.parquet
102
+ /dev/shm/data-files/hindi-data3/89.parquet
103
+ /dev/shm/data-files/hindi-data3/88.parquet
104
+ /dev/shm/data-files/hindi-data3/87.parquet
105
+ /dev/shm/data-files/hindi-data3/86.parquet
106
+ /dev/shm/data-files/hindi-data3/85.parquet
107
+ /dev/shm/data-files/hindi-data3/84.parquet
108
+ /dev/shm/data-files/hindi-data3/83.parquet
109
+ /dev/shm/data-files/hindi-data3/82.parquet
110
+ /dev/shm/data-files/hindi-data3/81.parquet
111
+ /dev/shm/data-files/hindi-data3/80.parquet
112
+ /dev/shm/data-files/hindi-data3/8.parquet
113
+ /dev/shm/data-files/hindi-data3/79.parquet
114
+ /dev/shm/data-files/hindi-data3/78.parquet
115
+ /dev/shm/data-files/hindi-data3/77.parquet
116
+ /dev/shm/data-files/hindi-data3/76.parquet
117
+ /dev/shm/data-files/hindi-data3/75.parquet
118
+ /dev/shm/data-files/hindi-data3/74.parquet
119
+ /dev/shm/data-files/hindi-data3/73.parquet
120
+ /dev/shm/data-files/hindi-data3/72.parquet
121
+ /dev/shm/data-files/hindi-data3/71.parquet
122
+ /dev/shm/data-files/hindi-data3/70.parquet
123
+ /dev/shm/data-files/hindi-data3/7.parquet
124
+ /dev/shm/data-files/hindi-data3/69.parquet
125
+ /dev/shm/data-files/hindi-data3/68.parquet
126
+ /dev/shm/data-files/hindi-data3/67.parquet
127
+ /dev/shm/data-files/hindi-data3/66.parquet
128
+ /dev/shm/data-files/hindi-data3/65.parquet
129
+ /dev/shm/data-files/hindi-data3/64.parquet
130
+ /dev/shm/data-files/hindi-data3/63.parquet
131
+ /dev/shm/data-files/hindi-data3/62.parquet
132
+ /dev/shm/data-files/hindi-data3/61.parquet
133
+ /dev/shm/data-files/hindi-data3/60.parquet
134
+ /dev/shm/data-files/hindi-data3/6.parquet
135
+ /dev/shm/data-files/hindi-data3/59.parquet
136
+ /dev/shm/data-files/hindi-data3/58.parquet
137
+ /dev/shm/data-files/hindi-data3/57.parquet
138
+ /dev/shm/data-files/hindi-data3/56.parquet
139
+ /dev/shm/data-files/hindi-data3/55.parquet
140
+ /dev/shm/data-files/hindi-data3/54.parquet
141
+ /dev/shm/data-files/hindi-data3/53.parquet
142
+ /dev/shm/data-files/hindi-data3/52.parquet
143
+ /dev/shm/data-files/hindi-data3/51.parquet
144
+ /dev/shm/data-files/hindi-data3/50.parquet
145
+ /dev/shm/data-files/hindi-data3/5.parquet
146
+ /dev/shm/data-files/hindi-data3/49.parquet
147
+ /dev/shm/data-files/hindi-data3/48.parquet
148
+ /dev/shm/data-files/hindi-data3/47.parquet
149
+ /dev/shm/data-files/hindi-data3/46.parquet
150
+ /dev/shm/data-files/hindi-data3/45.parquet
151
+ /dev/shm/data-files/hindi-data3/44.parquet
152
+ /dev/shm/data-files/hindi-data3/43.parquet
153
+ /dev/shm/data-files/hindi-data3/42.parquet
154
+ /dev/shm/data-files/hindi-data3/41.parquet
155
+ /dev/shm/data-files/hindi-data3/40.parquet
156
+ /dev/shm/data-files/hindi-data3/4.parquet
157
+ /dev/shm/data-files/hindi-data3/39.parquet
158
+ /dev/shm/data-files/hindi-data3/38.parquet
159
+ /dev/shm/data-files/hindi-data3/37.parquet
160
+ /dev/shm/data-files/hindi-data3/36.parquet
161
+ /dev/shm/data-files/hindi-data3/35.parquet
162
+ /dev/shm/data-files/hindi-data3/34.parquet
163
+ /dev/shm/data-files/hindi-data3/33.parquet
164
+ /dev/shm/data-files/hindi-data3/32.parquet
165
+ /dev/shm/data-files/hindi-data3/31.parquet
166
+ /dev/shm/data-files/hindi-data3/30.parquet
167
+ /dev/shm/data-files/hindi-data3/3.parquet
168
+ /dev/shm/data-files/hindi-data3/29.parquet
169
+ /dev/shm/data-files/hindi-data3/28.parquet
170
+ /dev/shm/data-files/hindi-data3/27.parquet
171
+ /dev/shm/data-files/hindi-data3/26.parquet
172
+ /dev/shm/data-files/hindi-data3/25.parquet
173
+ /dev/shm/data-files/hindi-data3/24.parquet
174
+ /dev/shm/data-files/hindi-data3/23.parquet
175
+ /dev/shm/data-files/hindi-data3/22.parquet
176
+ /dev/shm/data-files/hindi-data3/21.parquet
177
+ /dev/shm/data-files/hindi-data3/20.parquet
178
+ /dev/shm/data-files/hindi-data3/2.parquet
179
+ /dev/shm/data-files/hindi-data3/19.parquet
180
+ /dev/shm/data-files/hindi-data3/18.parquet
181
+ /dev/shm/data-files/hindi-data3/17.parquet
182
+ /dev/shm/data-files/hindi-data3/16.parquet
183
+ /dev/shm/data-files/hindi-data3/15.parquet
184
+ /dev/shm/data-files/hindi-data3/14.parquet
185
+ /dev/shm/data-files/hindi-data3/13.parquet
186
+ /dev/shm/data-files/hindi-data3/12.parquet
187
+ /dev/shm/data-files/hindi-data3/11.parquet
188
+ /dev/shm/data-files/hindi-data3/103.parquet
189
+ /dev/shm/data-files/hindi-data3/102.parquet
190
+ /dev/shm/data-files/hindi-data3/101.parquet
191
+ /dev/shm/data-files/hindi-data3/100.parquet
192
+ /dev/shm/data-files/hindi-data3/10.parquet
193
+ /dev/shm/data-files/hindi-data3/1.parquet
194
+ /dev/shm/data-files/hindi-data3/0.parquet
195
+ /dev/shm/data-files/hindi-data2/9.parquet
196
+ /dev/shm/data-files/hindi-data2/8.parquet
197
+ /dev/shm/data-files/hindi-data2/7.parquet
198
+ /dev/shm/data-files/hindi-data2/6.parquet
199
+ /dev/shm/data-files/hindi-data2/5.parquet
200
+ /dev/shm/data-files/hindi-data2/4.parquet
201
+ /dev/shm/data-files/hindi-data2/3.parquet
202
+ /dev/shm/data-files/hindi-data2/25.parquet
203
+ /dev/shm/data-files/hindi-data2/24.parquet
204
+ /dev/shm/data-files/hindi-data2/23.parquet
205
+ /dev/shm/data-files/hindi-data2/22.parquet
206
+ /dev/shm/data-files/hindi-data2/21.parquet
207
+ /dev/shm/data-files/hindi-data2/20.parquet
208
+ /dev/shm/data-files/hindi-data2/2.parquet
209
+ /dev/shm/data-files/hindi-data2/19.parquet
210
+ /dev/shm/data-files/hindi-data2/18.parquet
211
+ /dev/shm/data-files/hindi-data2/17.parquet
212
+ /dev/shm/data-files/hindi-data2/16.parquet
213
+ /dev/shm/data-files/hindi-data2/15.parquet
214
+ /dev/shm/data-files/hindi-data2/14.parquet
215
+ /dev/shm/data-files/hindi-data2/13.parquet
216
+ /dev/shm/data-files/hindi-data2/12.parquet
217
+ /dev/shm/data-files/hindi-data2/11.parquet
218
+ /dev/shm/data-files/hindi-data2/10.parquet
219
+ /dev/shm/data-files/hindi-data2/1.parquet
220
+ /dev/shm/data-files/hindi-data2/0.parquet
221
+ /dev/shm/data-files/hindi-data1/9.parquet
222
+ /dev/shm/data-files/hindi-data1/8.parquet
223
+ /dev/shm/data-files/hindi-data1/7.parquet
224
+ /dev/shm/data-files/hindi-data1/6.parquet
225
+ /dev/shm/data-files/hindi-data1/5.parquet
226
+ /dev/shm/data-files/hindi-data1/46.parquet
227
+ /dev/shm/data-files/hindi-data1/45.parquet
228
+ /dev/shm/data-files/hindi-data1/44.parquet
229
+ /dev/shm/data-files/hindi-data1/43.parquet
230
+ /dev/shm/data-files/hindi-data1/42.parquet
231
+ /dev/shm/data-files/hindi-data1/41.parquet
232
+ /dev/shm/data-files/hindi-data1/40.parquet
233
+ /dev/shm/data-files/hindi-data1/4.parquet
234
+ /dev/shm/data-files/hindi-data1/39.parquet
235
+ /dev/shm/data-files/hindi-data1/38.parquet
236
+ /dev/shm/data-files/hindi-data1/37.parquet
237
+ /dev/shm/data-files/hindi-data1/36.parquet
238
+ /dev/shm/data-files/hindi-data1/35.parquet
239
+ /dev/shm/data-files/hindi-data1/34.parquet
240
+ /dev/shm/data-files/hindi-data1/33.parquet
241
+ /dev/shm/data-files/hindi-data1/32.parquet
242
+ /dev/shm/data-files/hindi-data1/31.parquet
243
+ /dev/shm/data-files/hindi-data1/30.parquet
244
+ /dev/shm/data-files/hindi-data1/3.parquet
245
+ /dev/shm/data-files/hindi-data1/29.parquet
246
+ /dev/shm/data-files/hindi-data1/28.parquet
247
+ /dev/shm/data-files/hindi-data1/27.parquet
248
+ /dev/shm/data-files/hindi-data1/26.parquet
249
+ /dev/shm/data-files/hindi-data1/25.parquet
250
+ /dev/shm/data-files/hindi-data1/24.parquet
251
+ /dev/shm/data-files/hindi-data1/23.parquet
252
+ /dev/shm/data-files/hindi-data1/22.parquet
253
+ /dev/shm/data-files/hindi-data1/21.parquet
254
+ /dev/shm/data-files/hindi-data1/20.parquet
255
+ /dev/shm/data-files/hindi-data1/2.parquet
256
+ /dev/shm/data-files/hindi-data1/19.parquet
257
+ /dev/shm/data-files/hindi-data1/18.parquet
258
+ /dev/shm/data-files/hindi-data1/17.parquet
259
+ /dev/shm/data-files/hindi-data1/16.parquet
260
+ /dev/shm/data-files/hindi-data1/15.parquet
261
+ /dev/shm/data-files/hindi-data1/14.parquet
262
+ /dev/shm/data-files/hindi-data1/13.parquet
263
+ /dev/shm/data-files/hindi-data1/12.parquet
264
+ /dev/shm/data-files/hindi-data1/11.parquet
265
+ /dev/shm/data-files/hindi-data1/10.parquet
266
+ /dev/shm/data-files/hindi-data1/1.parquet
267
+ /dev/shm/data-files/hindi-data1/0.parquet
268
+ /dev/shm/data-files/english-data/9.parquet
269
+ /dev/shm/data-files/english-data/8.parquet
270
+ /dev/shm/data-files/english-data/7.parquet
271
+ /dev/shm/data-files/english-data/6.parquet
272
+ /dev/shm/data-files/english-data/57.parquet
273
+ /dev/shm/data-files/english-data/56.parquet
274
+ /dev/shm/data-files/english-data/55.parquet
275
+ /dev/shm/data-files/english-data/54.parquet
276
+ /dev/shm/data-files/english-data/53.parquet
277
+ /dev/shm/data-files/english-data/52.parquet
278
+ /dev/shm/data-files/english-data/51.parquet
279
+ /dev/shm/data-files/english-data/50.parquet
280
+ /dev/shm/data-files/english-data/5.parquet
281
+ /dev/shm/data-files/english-data/49.parquet
282
+ /dev/shm/data-files/english-data/48.parquet
283
+ /dev/shm/data-files/english-data/47.parquet
284
+ /dev/shm/data-files/english-data/46.parquet
285
+ /dev/shm/data-files/english-data/45.parquet
286
+ /dev/shm/data-files/english-data/44.parquet
287
+ /dev/shm/data-files/english-data/43.parquet
288
+ /dev/shm/data-files/english-data/42.parquet
289
+ /dev/shm/data-files/english-data/41.parquet
290
+ /dev/shm/data-files/english-data/40.parquet
291
+ /dev/shm/data-files/english-data/4.parquet
292
+ /dev/shm/data-files/english-data/39.parquet
293
+ /dev/shm/data-files/english-data/38.parquet
294
+ /dev/shm/data-files/english-data/37.parquet
295
+ /dev/shm/data-files/english-data/36.parquet
296
+ /dev/shm/data-files/english-data/35.parquet
297
+ /dev/shm/data-files/english-data/34.parquet
298
+ /dev/shm/data-files/english-data/33.parquet
299
+ /dev/shm/data-files/english-data/32.parquet
300
+ /dev/shm/data-files/english-data/31.parquet
301
+ /dev/shm/data-files/english-data/30.parquet
302
+ /dev/shm/data-files/english-data/3.parquet
303
+ /dev/shm/data-files/english-data/29.parquet
304
+ /dev/shm/data-files/english-data/28.parquet
305
+ /dev/shm/data-files/english-data/27.parquet
306
+ /dev/shm/data-files/english-data/26.parquet
307
+ /dev/shm/data-files/english-data/25.parquet
308
+ /dev/shm/data-files/english-data/24.parquet
309
+ /dev/shm/data-files/english-data/23.parquet
310
+ /dev/shm/data-files/english-data/22.parquet
311
+ /dev/shm/data-files/english-data/21.parquet
312
+ /dev/shm/data-files/english-data/20.parquet
313
+ /dev/shm/data-files/english-data/2.parquet
314
+ /dev/shm/data-files/english-data/19.parquet
315
+ /dev/shm/data-files/english-data/18.parquet
316
+ /dev/shm/data-files/english-data/17.parquet
317
+ /dev/shm/data-files/english-data/16.parquet
318
+ /dev/shm/data-files/english-data/15.parquet
319
+ /dev/shm/data-files/english-data/14.parquet
320
+ /dev/shm/data-files/english-data/13.parquet
321
+ /dev/shm/data-files/english-data/12.parquet
322
+ /dev/shm/data-files/english-data/11.parquet
323
+ /dev/shm/data-files/english-data/10.parquet
324
+ /dev/shm/data-files/english-data/1.parquet
325
+ /dev/shm/data-files/english-data/0.parquet
flax_model.msgpack CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:9a27a51818882c80dc60c00a6758070d78bd85e45a66c91a5c31cf6f491516c7
3
  size 962037499
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9d967d79b8d0e4b4a23656fec185bbb9e68940ad2e013e9a7b0f0f77b5c0a666
3
  size 962037499
muril-bigbird/flax_model.msgpack CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:92b4eb4b41a4fdd29fa296ca969236da8c1c0c923dab34bfff0448484f2a8228
3
  size 962037499
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9a27a51818882c80dc60c00a6758070d78bd85e45a66c91a5c31cf6f491516c7
3
  size 962037499
run_muril_bb_flax.sh CHANGED
@@ -5,6 +5,7 @@ python run_muril_bb_flax_mlm.py \
5
  --tokenizer_name muril-bigbird \
6
  --dtype bfloat16 \
7
  --pretokenized \
 
8
  --train_file "/dev/shm/data-files" \
9
  --validation_file "." \
10
  --max_seq_length 4096 \
@@ -20,9 +21,9 @@ python run_muril_bb_flax_mlm.py \
20
  --weight_decay 0.01 \
21
  --num_train_epochs 3 \
22
  --warmup_steps 0 \
23
- --logging_steps 500 \
24
- --save_steps 50000 \
25
  --save_total_limit 1 \
26
  --seed 18 \
27
  --push_to_hub \
28
- --run_name "en-ta-hi-te-from-50k"
 
5
  --tokenizer_name muril-bigbird \
6
  --dtype bfloat16 \
7
  --pretokenized \
8
+ --resume_from_checkpoint "checkpoint_100000" \
9
  --train_file "/dev/shm/data-files" \
10
  --validation_file "." \
11
  --max_seq_length 4096 \
 
21
  --weight_decay 0.01 \
22
  --num_train_epochs 3 \
23
  --warmup_steps 0 \
24
+ --logging_steps 1000 \
25
+ --save_steps 25000 \
26
  --save_total_limit 1 \
27
  --seed 18 \
28
  --push_to_hub \
29
+ --run_name "en-ta-hi-te-from-100k"
run_muril_bb_flax_mlm.py CHANGED
@@ -343,7 +343,7 @@ if __name__ == "__main__":
343
  files = p.glob("*/*.parquet")
344
  files = [str(x) for x in files]
345
  random.shuffle(files)
346
- files = files[50:100]
347
  datasets= load_dataset("parquet", data_files=files, split="train", cache_dir="/dev/shm")
348
  datasets = datasets.shuffle(seed=training_args.seed)
349
  datasets = datasets.train_test_split(test_size = 0.02, seed=training_args.seed)
 
343
  files = p.glob("*/*.parquet")
344
  files = [str(x) for x in files]
345
  random.shuffle(files)
346
+ files = files[100:150]
347
  datasets= load_dataset("parquet", data_files=files, split="train", cache_dir="/dev/shm")
348
  datasets = datasets.shuffle(seed=training_args.seed)
349
  datasets = datasets.train_test_split(test_size = 0.02, seed=training_args.seed)
wandb/debug-internal.log CHANGED
@@ -1 +1 @@
1
- run-20211107_204808-2ns3lo5n/logs/debug-internal.log
 
1
+ run-20211108_082107-2idnuxcc/logs/debug-internal.log
wandb/debug.log CHANGED
@@ -1 +1 @@
1
- run-20211107_204808-2ns3lo5n/logs/debug.log
 
1
+ run-20211108_082107-2idnuxcc/logs/debug.log
wandb/latest-run CHANGED
@@ -1 +1 @@
1
- run-20211107_204808-2ns3lo5n
 
1
+ run-20211108_082107-2idnuxcc
wandb/run-20211107_204808-2ns3lo5n/files/config.yaml CHANGED
@@ -23,6 +23,12 @@ _wandb:
23
  - 3
24
  - 11
25
  - 12
 
 
 
 
 
 
26
  3:
27
  - 13
28
  4: 3.8.10
 
23
  - 3
24
  - 11
25
  - 12
26
+ 2:
27
+ - 1
28
+ - 2
29
+ - 3
30
+ - 11
31
+ - 12
32
  3:
33
  - 13
34
  4: 3.8.10
wandb/run-20211107_204808-2ns3lo5n/files/output.log CHANGED
@@ -12484,3 +12484,5417 @@ tcmalloc: large alloc 2886115328 bytes == 0x5c1898000 @ 0x7fee676b5680 0x7fee67
12484
  [04:14:14] - INFO - absl - Saved checkpoint at ./checkpoint_100000
12485
  [04:14:14] - INFO - absl - Removing checkpoint at ./checkpoint_50000
12486
  Adding files tracked by Git LFS: ['checkpoint_100000']. This may take a bit of time if the files are large.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
12484
  [04:14:14] - INFO - absl - Saved checkpoint at ./checkpoint_100000
12485
  [04:14:14] - INFO - absl - Removing checkpoint at ./checkpoint_50000
12486
  Adding files tracked by Git LFS: ['checkpoint_100000']. This may take a bit of time if the files are large.
12487
+ [04:14:39] - WARNING - huggingface_hub.repository - Adding files tracked by Git LFS: ['checkpoint_100000']. This may take a bit of time if the files are large.
12488
+
12489
+
12490
+
12491
+
12492
+
12493
+
12494
+
12495
+
12496
+
12497
+
12498
+
12499
+
12500
+
12501
+
12502
+
12503
+
12504
+
12505
+
12506
+
12507
+
12508
+
12509
+
12510
+
12511
+
12512
+
12513
+
12514
+
12515
+
12516
+
12517
+
12518
+
12519
+
12520
+
12521
+
12522
+
12523
+
12524
+
12525
+
12526
+
12527
+
12528
+
12529
+
12530
+
12531
+
12532
+
12533
+
12534
+
12535
+
12536
+
12537
+
12538
+
12539
+
12540
+
12541
+
12542
+
12543
+
12544
+
12545
+
12546
+
12547
+
12548
+
12549
+
12550
+
12551
+
12552
+
12553
+
12554
+
12555
+
12556
+
12557
+
12558
+
12559
+
12560
+
12561
+
12562
+
12563
+
12564
+
12565
+
12566
+
12567
+
12568
+
12569
+
12570
+
12571
+
12572
+
12573
+
12574
+
12575
+
12576
+
12577
+
12578
+
12579
+
12580
+
12581
+
12582
+
12583
+
12584
+
12585
+
12586
+
12587
+
12588
+
12589
+
12590
+
12591
+
12592
+
12593
+
12594
+
12595
+
12596
+
12597
+
12598
+
12599
+
12600
+
12601
+
12602
+ Training...: 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 50503/61250 [7:29:37<8:37:19, 2.89s/it]
12603
+
12604
+
12605
+
12606
+
12607
+
12608
+
12609
+
12610
+
12611
+
12612
+
12613
+
12614
+
12615
+
12616
+
12617
+
12618
+
12619
+
12620
+
12621
+
12622
+
12623
+
12624
+
12625
+
12626
+
12627
+
12628
+
12629
+
12630
+
12631
+
12632
+
12633
+
12634
+
12635
+
12636
+
12637
+
12638
+
12639
+
12640
+
12641
+
12642
+
12643
+
12644
+
12645
+
12646
+
12647
+
12648
+
12649
+
12650
+
12651
+
12652
+
12653
+
12654
+
12655
+
12656
+
12657
+
12658
+
12659
+
12660
+
12661
+
12662
+
12663
+
12664
+
12665
+
12666
+
12667
+
12668
+
12669
+
12670
+
12671
+
12672
+
12673
+
12674
+
12675
+
12676
+
12677
+
12678
+
12679
+
12680
+
12681
+
12682
+
12683
+
12684
+
12685
+
12686
+
12687
+
12688
+
12689
+
12690
+
12691
+
12692
+
12693
+
12694
+
12695
+
12696
+
12697
+
12698
+
12699
+
12700
+
12701
+
12702
+
12703
+
12704
+
12705
+
12706
+
12707
+
12708
+
12709
+
12710
+
12711
+
12712
+
12713
+
12714
+
12715
+
12716
+ Training...: 83%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 51000/61250 [7:33:23<1:22:50, 2.06it/s]
12717
+
12718
+
12719
+
12720
+
12721
+
12722
+
12723
+
12724
+
12725
+
12726
+
12727
+
12728
+
12729
+
12730
+
12731
+
12732
+
12733
+
12734
+
12735
+
12736
+
12737
+
12738
+
12739
+
12740
+
12741
+
12742
+
12743
+
12744
+
12745
+
12746
+
12747
+
12748
+
12749
+
12750
+
12751
+
12752
+
12753
+
12754
+
12755
+
12756
+
12757
+
12758
+
12759
+
12760
+
12761
+
12762
+
12763
+
12764
+
12765
+
12766
+
12767
+
12768
+
12769
+
12770
+
12771
+
12772
+
12773
+
12774
+
12775
+
12776
+
12777
+
12778
+
12779
+
12780
+
12781
+
12782
+
12783
+
12784
+
12785
+
12786
+
12787
+
12788
+
12789
+
12790
+
12791
+
12792
+
12793
+
12794
+
12795
+
12796
+
12797
+
12798
+
12799
+
12800
+
12801
+
12802
+
12803
+
12804
+
12805
+
12806
+
12807
+
12808
+
12809
+
12810
+
12811
+
12812
+
12813
+
12814
+
12815
+
12816
+
12817
+
12818
+
12819
+
12820
+
12821
+
12822
+
12823
+
12824
+
12825
+
12826
+
12827
+
12828
+
12829
+
12830
+ Training...: 84%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 51500/61250 [7:37:26<1:18:48, 2.06it/s]
12831
+
12832
+
12833
+
12834
+
12835
+
12836
+
12837
+
12838
+
12839
+
12840
+
12841
+
12842
+
12843
+
12844
+
12845
+
12846
+
12847
+
12848
+
12849
+
12850
+
12851
+
12852
+
12853
+
12854
+
12855
+
12856
+
12857
+
12858
+
12859
+
12860
+
12861
+
12862
+
12863
+
12864
+
12865
+
12866
+
12867
+
12868
+
12869
+
12870
+
12871
+
12872
+
12873
+
12874
+
12875
+
12876
+
12877
+
12878
+
12879
+
12880
+
12881
+
12882
+
12883
+
12884
+
12885
+
12886
+
12887
+
12888
+
12889
+
12890
+
12891
+
12892
+
12893
+
12894
+
12895
+
12896
+
12897
+
12898
+
12899
+
12900
+
12901
+
12902
+
12903
+
12904
+
12905
+
12906
+
12907
+
12908
+
12909
+
12910
+
12911
+
12912
+
12913
+
12914
+
12915
+
12916
+
12917
+
12918
+
12919
+
12920
+
12921
+
12922
+
12923
+
12924
+
12925
+
12926
+
12927
+
12928
+
12929
+
12930
+
12931
+
12932
+
12933
+
12934
+
12935
+
12936
+
12937
+
12938
+
12939
+
12940
+
12941
+
12942
+
12943
+
12944
+ Training...: 85%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 52000/61250 [7:41:29<1:14:47, 2.06it/s]
12945
+
12946
+
12947
+
12948
+
12949
+
12950
+
12951
+
12952
+
12953
+
12954
+
12955
+
12956
+
12957
+
12958
+
12959
+
12960
+
12961
+
12962
+
12963
+
12964
+
12965
+
12966
+
12967
+
12968
+
12969
+
12970
+
12971
+
12972
+
12973
+
12974
+
12975
+
12976
+
12977
+
12978
+
12979
+
12980
+
12981
+
12982
+
12983
+
12984
+
12985
+
12986
+
12987
+
12988
+
12989
+
12990
+
12991
+
12992
+
12993
+
12994
+
12995
+
12996
+
12997
+
12998
+
12999
+
13000
+
13001
+
13002
+
13003
+
13004
+
13005
+
13006
+
13007
+
13008
+
13009
+
13010
+
13011
+
13012
+
13013
+
13014
+
13015
+
13016
+
13017
+
13018
+
13019
+
13020
+
13021
+
13022
+
13023
+
13024
+
13025
+
13026
+
13027
+
13028
+
13029
+
13030
+
13031
+
13032
+
13033
+
13034
+
13035
+
13036
+
13037
+
13038
+
13039
+
13040
+
13041
+
13042
+
13043
+
13044
+
13045
+
13046
+
13047
+
13048
+
13049
+
13050
+
13051
+
13052
+
13053
+
13054
+
13055
+
13056
+
13057
+
13058
+
13059
+ Training...: 86%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 52500/61250 [7:45:32<1:10:43, 2.06it/s]
13060
+
13061
+
13062
+
13063
+
13064
+
13065
+
13066
+
13067
+
13068
+
13069
+
13070
+
13071
+
13072
+
13073
+
13074
+
13075
+
13076
+
13077
+
13078
+
13079
+
13080
+
13081
+
13082
+
13083
+
13084
+
13085
+
13086
+
13087
+
13088
+
13089
+
13090
+
13091
+
13092
+
13093
+
13094
+
13095
+
13096
+
13097
+
13098
+
13099
+
13100
+
13101
+
13102
+
13103
+
13104
+
13105
+
13106
+
13107
+
13108
+
13109
+
13110
+
13111
+
13112
+
13113
+
13114
+
13115
+
13116
+
13117
+
13118
+
13119
+
13120
+
13121
+
13122
+
13123
+
13124
+
13125
+
13126
+
13127
+
13128
+
13129
+
13130
+
13131
+
13132
+
13133
+
13134
+
13135
+
13136
+
13137
+
13138
+
13139
+
13140
+
13141
+
13142
+
13143
+
13144
+
13145
+
13146
+
13147
+
13148
+
13149
+
13150
+
13151
+
13152
+
13153
+
13154
+
13155
+
13156
+
13157
+
13158
+
13159
+
13160
+
13161
+
13162
+
13163
+
13164
+
13165
+
13166
+
13167
+
13168
+
13169
+
13170
+
13171
+
13172
+
13173
+
13174
+
13175
+ Training...: 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████��██████████████████▊ | 53019/61250 [7:49:53<23:27, 5.85it/s]
13176
+
13177
+
13178
+
13179
+
13180
+
13181
+
13182
+
13183
+
13184
+
13185
+
13186
+
13187
+
13188
+
13189
+
13190
+
13191
+
13192
+
13193
+
13194
+
13195
+
13196
+
13197
+
13198
+
13199
+
13200
+
13201
+
13202
+
13203
+
13204
+
13205
+
13206
+
13207
+
13208
+
13209
+
13210
+
13211
+
13212
+
13213
+
13214
+
13215
+
13216
+
13217
+
13218
+
13219
+
13220
+
13221
+
13222
+
13223
+
13224
+
13225
+
13226
+
13227
+
13228
+
13229
+
13230
+
13231
+
13232
+
13233
+
13234
+
13235
+
13236
+
13237
+
13238
+
13239
+
13240
+
13241
+
13242
+
13243
+
13244
+
13245
+
13246
+
13247
+
13248
+
13249
+
13250
+
13251
+
13252
+
13253
+
13254
+
13255
+
13256
+
13257
+
13258
+
13259
+
13260
+
13261
+
13262
+
13263
+
13264
+
13265
+
13266
+
13267
+
13268
+
13269
+
13270
+
13271
+
13272
+
13273
+
13274
+
13275
+
13276
+
13277
+
13278
+
13279
+
13280
+
13281
+
13282
+
13283
+
13284
+
13285
+
13286
+
13287
+
13288
+
13289
+
13290
+ Training...: 87%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 53519/61250 [7:53:56<21:59, 5.86it/s]
13291
+
13292
+
13293
+
13294
+
13295
+
13296
+
13297
+
13298
+
13299
+
13300
+
13301
+
13302
+
13303
+
13304
+
13305
+
13306
+
13307
+
13308
+
13309
+
13310
+
13311
+
13312
+
13313
+
13314
+
13315
+
13316
+
13317
+
13318
+
13319
+
13320
+
13321
+
13322
+
13323
+
13324
+
13325
+
13326
+
13327
+
13328
+
13329
+
13330
+
13331
+
13332
+
13333
+
13334
+
13335
+
13336
+
13337
+
13338
+
13339
+
13340
+
13341
+
13342
+
13343
+
13344
+
13345
+
13346
+
13347
+
13348
+
13349
+
13350
+
13351
+
13352
+
13353
+
13354
+
13355
+
13356
+
13357
+
13358
+
13359
+
13360
+
13361
+
13362
+
13363
+
13364
+
13365
+
13366
+
13367
+
13368
+
13369
+
13370
+
13371
+
13372
+
13373
+
13374
+
13375
+
13376
+
13377
+
13378
+
13379
+
13380
+
13381
+
13382
+
13383
+
13384
+
13385
+
13386
+
13387
+
13388
+
13389
+
13390
+
13391
+
13392
+
13393
+
13394
+
13395
+
13396
+
13397
+
13398
+
13399
+
13400
+
13401
+
13402
+
13403
+
13404
+ Training...: 88%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 54000/61250 [7:57:42<58:36, 2.06it/s]
13405
+
13406
+
13407
+
13408
+
13409
+
13410
+
13411
+
13412
+
13413
+
13414
+
13415
+
13416
+
13417
+
13418
+
13419
+
13420
+
13421
+
13422
+
13423
+
13424
+
13425
+
13426
+
13427
+
13428
+
13429
+
13430
+
13431
+
13432
+
13433
+
13434
+
13435
+
13436
+
13437
+
13438
+
13439
+
13440
+
13441
+
13442
+
13443
+
13444
+
13445
+
13446
+
13447
+
13448
+
13449
+
13450
+
13451
+
13452
+
13453
+
13454
+
13455
+
13456
+
13457
+
13458
+
13459
+
13460
+
13461
+
13462
+
13463
+
13464
+
13465
+
13466
+
13467
+
13468
+
13469
+
13470
+
13471
+
13472
+
13473
+
13474
+
13475
+
13476
+
13477
+
13478
+
13479
+
13480
+
13481
+
13482
+
13483
+
13484
+
13485
+
13486
+
13487
+
13488
+
13489
+
13490
+
13491
+
13492
+
13493
+
13494
+
13495
+
13496
+
13497
+
13498
+
13499
+
13500
+
13501
+
13502
+
13503
+
13504
+
13505
+
13506
+
13507
+
13508
+
13509
+
13510
+
13511
+
13512
+
13513
+
13514
+
13515
+
13516
+
13517
+
13518
+
13519
+ Training...: 89%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 54500/61250 [8:01:45<54:34, 2.06it/s]
13520
+
13521
+
13522
+
13523
+
13524
+
13525
+
13526
+
13527
+
13528
+
13529
+
13530
+
13531
+
13532
+
13533
+
13534
+
13535
+
13536
+
13537
+
13538
+
13539
+
13540
+
13541
+
13542
+
13543
+
13544
+
13545
+
13546
+
13547
+
13548
+
13549
+
13550
+
13551
+
13552
+
13553
+
13554
+
13555
+
13556
+
13557
+
13558
+
13559
+
13560
+
13561
+
13562
+
13563
+
13564
+
13565
+
13566
+
13567
+
13568
+
13569
+
13570
+
13571
+
13572
+
13573
+
13574
+
13575
+
13576
+
13577
+
13578
+
13579
+
13580
+
13581
+
13582
+
13583
+
13584
+
13585
+
13586
+
13587
+
13588
+
13589
+
13590
+
13591
+
13592
+
13593
+
13594
+
13595
+
13596
+
13597
+
13598
+
13599
+
13600
+
13601
+
13602
+
13603
+
13604
+
13605
+
13606
+
13607
+
13608
+
13609
+
13610
+
13611
+
13612
+
13613
+
13614
+
13615
+
13616
+
13617
+
13618
+
13619
+
13620
+
13621
+
13622
+
13623
+
13624
+
13625
+
13626
+
13627
+
13628
+
13629
+
13630
+
13631
+
13632
+
13633
+
13634
+
13635
+ Training...: 90%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 55000/61250 [8:06:05<50:30, 2.06it/s]
13636
+ Step... (105000 | Loss: 1.4761005640029907, Learning Rate: 3.5034012398682535e-05)
13637
+
13638
+
13639
+
13640
+
13641
+
13642
+
13643
+
13644
+
13645
+
13646
+
13647
+
13648
+
13649
+
13650
+
13651
+
13652
+
13653
+
13654
+
13655
+
13656
+
13657
+
13658
+
13659
+
13660
+
13661
+
13662
+
13663
+
13664
+
13665
+
13666
+
13667
+
13668
+
13669
+
13670
+
13671
+
13672
+
13673
+
13674
+
13675
+
13676
+
13677
+
13678
+
13679
+
13680
+
13681
+
13682
+
13683
+
13684
+
13685
+
13686
+
13687
+
13688
+
13689
+
13690
+
13691
+
13692
+
13693
+
13694
+
13695
+
13696
+
13697
+
13698
+
13699
+
13700
+
13701
+
13702
+
13703
+
13704
+
13705
+
13706
+
13707
+
13708
+
13709
+
13710
+
13711
+
13712
+
13713
+
13714
+
13715
+
13716
+
13717
+
13718
+
13719
+
13720
+
13721
+
13722
+
13723
+
13724
+
13725
+
13726
+
13727
+
13728
+
13729
+
13730
+
13731
+
13732
+
13733
+
13734
+
13735
+
13736
+
13737
+
13738
+
13739
+
13740
+
13741
+
13742
+
13743
+
13744
+
13745
+
13746
+
13747
+
13748
+
13749
+
13750
+
13751
+
13752
+
13753
+
13754
+
13755
+
13756
+
13757
+
13758
+
13759
+
13760
+
13761
+
13762
+
13763
+
13764
+
13765
+
13766
+
13767
+
13768
+
13769
+
13770
+
13771
+
13772
+
13773
+
13774
+
13775
+
13776
+
13777
+
13778
+
13779
+
13780
+
13781
+
13782
+
13783
+
13784
+
13785
+
13786
+
13787
+
13788
+
13789
+
13790
+
13791
+
13792
+
13793
+
13794
+
13795
+
13796
+
13797
+
13798
+
13799
+
13800
+
13801
+
13802
+
13803
+
13804
+
13805
+
13806
+
13807
+
13808
+
13809
+
13810
+
13811
+
13812
+
13813
+
13814
+
13815
+
13816
+
13817
+
13818
+
13819
+
13820
+
13821
+
13822
+
13823
+
13824
+
13825
+
13826
+
13827
+
13828
+
13829
+
13830
+
13831
+
13832
+
13833
+
13834
+
13835
+
13836
+
13837
+
13838
+
13839
+ Training...: 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 55517/61250 [8:13:17<20:53, 4.58it/s]
13840
+
13841
+
13842
+
13843
+
13844
+
13845
+
13846
+
13847
+
13848
+
13849
+
13850
+
13851
+
13852
+
13853
+
13854
+
13855
+
13856
+
13857
+
13858
+
13859
+
13860
+
13861
+
13862
+
13863
+
13864
+
13865
+
13866
+
13867
+
13868
+
13869
+
13870
+
13871
+
13872
+
13873
+
13874
+
13875
+
13876
+
13877
+
13878
+
13879
+
13880
+
13881
+
13882
+
13883
+
13884
+
13885
+
13886
+
13887
+
13888
+
13889
+
13890
+
13891
+
13892
+
13893
+
13894
+
13895
+
13896
+
13897
+
13898
+
13899
+
13900
+
13901
+
13902
+
13903
+
13904
+
13905
+
13906
+
13907
+
13908
+
13909
+
13910
+
13911
+
13912
+
13913
+
13914
+
13915
+
13916
+
13917
+
13918
+
13919
+
13920
+
13921
+
13922
+
13923
+
13924
+
13925
+
13926
+
13927
+
13928
+
13929
+
13930
+
13931
+
13932
+
13933
+
13934
+
13935
+
13936
+
13937
+
13938
+
13939
+
13940
+
13941
+
13942
+
13943
+
13944
+
13945
+
13946
+
13947
+
13948
+
13949
+
13950
+
13951
+
13952
+
13953
+
13954
+ Training...: 91%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████��██████████████████████ | 56011/61250 [8:17:19<47:34, 1.84it/s]
13955
+
13956
+
13957
+
13958
+
13959
+
13960
+
13961
+
13962
+
13963
+
13964
+
13965
+
13966
+
13967
+
13968
+
13969
+
13970
+
13971
+
13972
+
13973
+
13974
+
13975
+
13976
+
13977
+
13978
+
13979
+
13980
+
13981
+
13982
+
13983
+
13984
+
13985
+
13986
+
13987
+
13988
+
13989
+
13990
+
13991
+
13992
+
13993
+
13994
+
13995
+
13996
+
13997
+
13998
+
13999
+
14000
+
14001
+
14002
+
14003
+
14004
+
14005
+
14006
+
14007
+
14008
+
14009
+
14010
+
14011
+
14012
+
14013
+
14014
+
14015
+
14016
+
14017
+
14018
+
14019
+
14020
+
14021
+
14022
+
14023
+
14024
+
14025
+
14026
+
14027
+
14028
+
14029
+
14030
+
14031
+
14032
+
14033
+
14034
+
14035
+
14036
+
14037
+
14038
+
14039
+
14040
+
14041
+
14042
+
14043
+
14044
+
14045
+
14046
+
14047
+
14048
+
14049
+
14050
+
14051
+
14052
+
14053
+
14054
+
14055
+
14056
+
14057
+
14058
+
14059
+
14060
+
14061
+
14062
+
14063
+
14064
+
14065
+
14066
+
14067
+
14068
+ Training...: 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 56500/61250 [8:21:05<38:23, 2.06it/s]
14069
+
14070
+
14071
+
14072
+
14073
+
14074
+
14075
+
14076
+
14077
+
14078
+
14079
+
14080
+
14081
+
14082
+
14083
+
14084
+
14085
+
14086
+
14087
+
14088
+
14089
+
14090
+
14091
+
14092
+
14093
+
14094
+
14095
+
14096
+
14097
+
14098
+
14099
+
14100
+
14101
+
14102
+
14103
+
14104
+
14105
+
14106
+
14107
+
14108
+
14109
+
14110
+
14111
+
14112
+
14113
+
14114
+
14115
+
14116
+
14117
+
14118
+
14119
+
14120
+
14121
+
14122
+
14123
+
14124
+
14125
+
14126
+
14127
+
14128
+
14129
+
14130
+
14131
+
14132
+
14133
+
14134
+
14135
+
14136
+
14137
+
14138
+
14139
+
14140
+
14141
+
14142
+
14143
+
14144
+
14145
+
14146
+
14147
+
14148
+
14149
+
14150
+
14151
+
14152
+
14153
+
14154
+
14155
+
14156
+
14157
+
14158
+
14159
+
14160
+
14161
+
14162
+
14163
+
14164
+
14165
+
14166
+
14167
+
14168
+
14169
+
14170
+
14171
+
14172
+
14173
+
14174
+
14175
+
14176
+
14177
+
14178
+
14179
+
14180
+
14181
+
14182
+
14183
+ Training...: 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 57000/61250 [8:25:09<34:22, 2.06it/s]
14184
+
14185
+
14186
+
14187
+
14188
+
14189
+
14190
+
14191
+
14192
+
14193
+
14194
+
14195
+
14196
+
14197
+
14198
+
14199
+
14200
+
14201
+
14202
+
14203
+
14204
+
14205
+
14206
+
14207
+
14208
+
14209
+
14210
+
14211
+
14212
+
14213
+
14214
+
14215
+
14216
+
14217
+
14218
+
14219
+
14220
+
14221
+
14222
+
14223
+
14224
+
14225
+
14226
+
14227
+
14228
+
14229
+
14230
+
14231
+
14232
+
14233
+
14234
+
14235
+
14236
+
14237
+
14238
+
14239
+
14240
+
14241
+
14242
+
14243
+
14244
+
14245
+
14246
+
14247
+
14248
+
14249
+
14250
+
14251
+
14252
+
14253
+
14254
+
14255
+
14256
+
14257
+
14258
+
14259
+
14260
+
14261
+
14262
+
14263
+
14264
+
14265
+
14266
+
14267
+
14268
+
14269
+
14270
+
14271
+
14272
+
14273
+
14274
+
14275
+
14276
+
14277
+
14278
+
14279
+
14280
+
14281
+
14282
+
14283
+
14284
+
14285
+
14286
+
14287
+
14288
+
14289
+
14290
+
14291
+
14292
+
14293
+
14294
+
14295
+
14296
+
14297
+ Training...: 94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 57500/61250 [8:29:12<30:19, 2.06it/s]
14298
+
14299
+
14300
+
14301
+
14302
+
14303
+
14304
+
14305
+
14306
+
14307
+
14308
+
14309
+
14310
+
14311
+
14312
+
14313
+
14314
+
14315
+
14316
+
14317
+
14318
+
14319
+
14320
+
14321
+
14322
+
14323
+
14324
+
14325
+
14326
+
14327
+
14328
+
14329
+
14330
+
14331
+
14332
+
14333
+
14334
+
14335
+
14336
+
14337
+
14338
+
14339
+
14340
+
14341
+
14342
+
14343
+
14344
+
14345
+
14346
+
14347
+
14348
+
14349
+
14350
+
14351
+
14352
+
14353
+
14354
+
14355
+
14356
+
14357
+
14358
+
14359
+
14360
+
14361
+
14362
+
14363
+
14364
+
14365
+
14366
+
14367
+
14368
+
14369
+
14370
+
14371
+
14372
+
14373
+
14374
+
14375
+
14376
+
14377
+
14378
+
14379
+
14380
+
14381
+
14382
+
14383
+
14384
+
14385
+
14386
+
14387
+
14388
+
14389
+
14390
+
14391
+
14392
+
14393
+
14394
+
14395
+
14396
+
14397
+
14398
+
14399
+
14400
+
14401
+
14402
+
14403
+
14404
+
14405
+
14406
+
14407
+
14408
+
14409
+
14410
+
14411
+ Training...: 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 58000/61250 [8:33:15<26:16, 2.06it/s]
14412
+
14413
+
14414
+
14415
+
14416
+
14417
+
14418
+
14419
+
14420
+
14421
+
14422
+
14423
+
14424
+
14425
+
14426
+
14427
+
14428
+
14429
+
14430
+
14431
+
14432
+
14433
+
14434
+
14435
+
14436
+
14437
+
14438
+
14439
+
14440
+
14441
+
14442
+
14443
+
14444
+
14445
+
14446
+
14447
+
14448
+
14449
+
14450
+
14451
+
14452
+
14453
+
14454
+
14455
+
14456
+
14457
+
14458
+
14459
+
14460
+
14461
+
14462
+
14463
+
14464
+
14465
+
14466
+
14467
+
14468
+
14469
+
14470
+
14471
+
14472
+
14473
+
14474
+
14475
+
14476
+
14477
+
14478
+
14479
+
14480
+
14481
+
14482
+
14483
+
14484
+
14485
+
14486
+
14487
+
14488
+
14489
+
14490
+
14491
+
14492
+
14493
+
14494
+
14495
+
14496
+
14497
+
14498
+
14499
+
14500
+
14501
+
14502
+
14503
+
14504
+
14505
+
14506
+
14507
+
14508
+
14509
+
14510
+
14511
+
14512
+
14513
+
14514
+
14515
+
14516
+
14517
+
14518
+
14519
+
14520
+
14521
+
14522
+
14523
+
14524
+
14525
+ Training...: 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 58500/61250 [8:37:18<22:13, 2.06it/s]
14526
+
14527
+
14528
+
14529
+
14530
+
14531
+
14532
+
14533
+
14534
+
14535
+
14536
+
14537
+
14538
+
14539
+
14540
+
14541
+
14542
+
14543
+
14544
+
14545
+
14546
+
14547
+
14548
+
14549
+
14550
+
14551
+
14552
+
14553
+
14554
+
14555
+
14556
+
14557
+
14558
+
14559
+
14560
+
14561
+
14562
+
14563
+
14564
+
14565
+
14566
+
14567
+
14568
+
14569
+
14570
+
14571
+
14572
+
14573
+
14574
+
14575
+
14576
+
14577
+
14578
+
14579
+
14580
+
14581
+
14582
+
14583
+
14584
+
14585
+
14586
+
14587
+
14588
+
14589
+
14590
+
14591
+
14592
+
14593
+
14594
+
14595
+
14596
+
14597
+
14598
+
14599
+
14600
+
14601
+
14602
+
14603
+
14604
+
14605
+
14606
+
14607
+
14608
+
14609
+
14610
+
14611
+
14612
+
14613
+
14614
+
14615
+
14616
+
14617
+
14618
+
14619
+
14620
+
14621
+
14622
+
14623
+
14624
+
14625
+
14626
+
14627
+
14628
+
14629
+
14630
+
14631
+
14632
+
14633
+
14634
+
14635
+
14636
+
14637
+
14638
+
14639
+ Training...: 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 59000/61250 [8:41:21<18:10, 2.06it/s]
14640
+
14641
+
14642
+
14643
+
14644
+
14645
+
14646
+
14647
+
14648
+
14649
+
14650
+
14651
+
14652
+
14653
+
14654
+
14655
+
14656
+
14657
+
14658
+
14659
+
14660
+
14661
+
14662
+
14663
+
14664
+
14665
+
14666
+
14667
+
14668
+
14669
+
14670
+
14671
+
14672
+
14673
+
14674
+
14675
+
14676
+
14677
+
14678
+
14679
+
14680
+
14681
+
14682
+
14683
+
14684
+
14685
+
14686
+
14687
+
14688
+
14689
+
14690
+
14691
+
14692
+
14693
+
14694
+
14695
+
14696
+
14697
+
14698
+
14699
+
14700
+
14701
+
14702
+
14703
+
14704
+
14705
+
14706
+
14707
+
14708
+
14709
+
14710
+
14711
+
14712
+
14713
+
14714
+
14715
+
14716
+
14717
+
14718
+
14719
+
14720
+
14721
+
14722
+
14723
+
14724
+
14725
+
14726
+
14727
+
14728
+
14729
+
14730
+
14731
+
14732
+
14733
+
14734
+
14735
+
14736
+
14737
+
14738
+
14739
+
14740
+
14741
+
14742
+
14743
+
14744
+
14745
+
14746
+
14747
+
14748
+
14749
+
14750
+
14751
+
14752
+
14753
+
14754
+ Training...: 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 59525/61250 [8:45:43<02:52, 10.00it/s]
14755
+
14756
+
14757
+
14758
+
14759
+
14760
+
14761
+
14762
+
14763
+
14764
+
14765
+
14766
+
14767
+
14768
+
14769
+
14770
+
14771
+
14772
+
14773
+
14774
+
14775
+
14776
+
14777
+
14778
+
14779
+
14780
+
14781
+
14782
+
14783
+
14784
+
14785
+
14786
+
14787
+
14788
+
14789
+
14790
+
14791
+
14792
+
14793
+
14794
+
14795
+
14796
+
14797
+
14798
+
14799
+
14800
+
14801
+
14802
+
14803
+
14804
+
14805
+
14806
+
14807
+
14808
+
14809
+
14810
+
14811
+
14812
+
14813
+
14814
+
14815
+
14816
+
14817
+
14818
+
14819
+
14820
+
14821
+
14822
+
14823
+
14824
+
14825
+
14826
+
14827
+
14828
+
14829
+
14830
+
14831
+
14832
+
14833
+
14834
+
14835
+
14836
+
14837
+
14838
+
14839
+
14840
+
14841
+
14842
+
14843
+
14844
+
14845
+
14846
+
14847
+
14848
+
14849
+
14850
+
14851
+
14852
+
14853
+
14854
+
14855
+
14856
+
14857
+
14858
+
14859
+
14860
+
14861
+
14862
+
14863
+
14864
+
14865
+
14866
+
14867
+
14868
+ Training...: 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 60000/61250 [8:49:28<10:06, 2.06it/s]
14869
+ Training...: 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 60000/61250 [8:49:44<10:06, 2.06it/s]
14870
+
14871
+
14872
+
14873
+
14874
+
14875
+
14876
+
14877
+
14878
+
14879
+
14880
+
14881
+
14882
+
14883
+
14884
+
14885
+
14886
+
14887
+
14888
+
14889
+
14890
+
14891
+
14892
+
14893
+
14894
+
14895
+
14896
+
14897
+
14898
+
14899
+
14900
+
14901
+
14902
+
14903
+
14904
+
14905
+
14906
+
14907
+
14908
+
14909
+
14910
+
14911
+
14912
+
14913
+
14914
+
14915
+
14916
+
14917
+
14918
+
14919
+
14920
+
14921
+
14922
+
14923
+
14924
+
14925
+
14926
+
14927
+
14928
+
14929
+
14930
+
14931
+
14932
+
14933
+
14934
+
14935
+
14936
+
14937
+
14938
+
14939
+
14940
+
14941
+
14942
+
14943
+
14944
+
14945
+
14946
+
14947
+
14948
+
14949
+
14950
+
14951
+
14952
+
14953
+
14954
+
14955
+
14956
+
14957
+
14958
+
14959
+
14960
+
14961
+
14962
+
14963
+
14964
+
14965
+
14966
+
14967
+
14968
+
14969
+
14970
+
14971
+
14972
+
14973
+
14974
+
14975
+
14976
+
14977
+
14978
+
14979
+
14980
+
14981
+
14982
+
14983
+
14984
+
14985
+
14986
+
14987
+
14988
+
14989
+
14990
+
14991
+
14992
+
14993
+
14994
+
14995
+
14996
+
14997
+
14998
+
14999
+
15000
+
15001
+
15002
+
15003
+
15004
+
15005
+
15006
+
15007
+
15008
+
15009
+
15010
+
15011
+
15012
+
15013
+
15014
+
15015
+
15016
+
15017
+
15018
+
15019
+
15020
+
15021
+
15022
+
15023
+
15024
+
15025
+
15026
+
15027
+
15028
+
15029
+
15030
+
15031
+
15032
+
15033
+
15034
+
15035
+
15036
+
15037
+
15038
+
15039
+
15040
+
15041
+
15042
+
15043
+
15044
+
15045
+
15046
+
15047
+
15048
+
15049
+
15050
+
15051
+
15052
+
15053
+
15054
+
15055
+
15056
+
15057
+
15058
+
15059
+
15060
+
15061
+
15062
+
15063
+
15064
+
15065
+
15066
+
15067
+
15068
+
15069
+
15070
+
15071
+
15072
+
15073
+ Training...: 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 60521/61250 [8:56:56<01:41, 7.15it/s]
15074
+
15075
+
15076
+
15077
+
15078
+
15079
+
15080
+
15081
+
15082
+
15083
+
15084
+
15085
+
15086
+
15087
+
15088
+
15089
+
15090
+
15091
+
15092
+
15093
+
15094
+
15095
+
15096
+
15097
+
15098
+
15099
+
15100
+
15101
+
15102
+
15103
+
15104
+
15105
+
15106
+
15107
+
15108
+
15109
+
15110
+
15111
+
15112
+
15113
+
15114
+
15115
+
15116
+
15117
+
15118
+
15119
+
15120
+
15121
+
15122
+
15123
+
15124
+
15125
+
15126
+
15127
+
15128
+
15129
+
15130
+
15131
+
15132
+
15133
+
15134
+
15135
+
15136
+
15137
+
15138
+
15139
+
15140
+
15141
+
15142
+
15143
+
15144
+
15145
+
15146
+
15147
+
15148
+
15149
+
15150
+
15151
+
15152
+
15153
+
15154
+
15155
+
15156
+
15157
+
15158
+
15159
+
15160
+
15161
+
15162
+
15163
+
15164
+
15165
+
15166
+
15167
+
15168
+
15169
+
15170
+
15171
+
15172
+
15173
+
15174
+
15175
+
15176
+
15177
+
15178
+
15179
+
15180
+
15181
+
15182
+
15183
+
15184
+
15185
+
15186
+
15187
+ Training...: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏| 61000/61250 [9:00:42<02:01, 2.06it/s]
15188
+
15189
+
15190
+
15191
+
15192
+
15193
+
15194
+
15195
+
15196
+
15197
+
15198
+
15199
+
15200
+
15201
+
15202
+
15203
+
15204
+
15205
+
15206
+
15207
+
15208
+
15209
+
15210
+
15211
+
15212
+
15213
+
15214
+
15215
+
15216
+
15217
+
15218
+
15219
+
15220
+
15221
+
15222
+
15223
+
15224
+
15225
+
15226
+
15227
+
15228
+
15229
+
15230
+
15231
+
15232
+
15233
+
15234
+
15235
+
15236
+
15237
+
15238
+
15239
+
15240
+
15241
+ Step... (110000 | Loss: 1.404970645904541, Acc: 0.7204243540763855): 33%|████████████████████████████████████████████▎ | 1/3 [9:03:51<18:07:42, 32631.03s/it]
15242
+
15243
+
15244
+
15245
+
15246
+
15247
+
15248
+
15249
+
15250
+
15251
+
15252
+
15253
+
15254
+
15255
+
15256
+
15257
+
15258
+
15259
+
15260
+
15261
+
15262
+
15263
+
15264
+
15265
+
15266
+
15267
+
15268
+
15269
+
15270
+
15271
+
15272
+
15273
+
15274
+
15275
+
15276
+
15277
+
15278
+
15279
+
15280
+
15281
+
15282
+
15283
+
15284
+
15285
+
15286
+
15287
+
15288
+
15289
+
15290
+
15291
+
15292
+
15293
+
15294
+
15295
+ Training...: 0%|▊ | 251/61250 [02:02<88:28:04, 5.22s/it]
15296
+
15297
+
15298
+
15299
+
15300
+
15301
+
15302
+
15303
+
15304
+
15305
+
15306
+
15307
+
15308
+
15309
+
15310
+
15311
+
15312
+
15313
+
15314
+
15315
+
15316
+
15317
+
15318
+
15319
+
15320
+
15321
+
15322
+
15323
+
15324
+
15325
+
15326
+
15327
+
15328
+
15329
+
15330
+
15331
+
15332
+
15333
+
15334
+
15335
+
15336
+
15337
+
15338
+
15339
+
15340
+
15341
+
15342
+
15343
+
15344
+
15345
+
15346
+
15347
+
15348
+
15349
+
15350
+
15351
+
15352
+
15353
+
15354
+
15355
+
15356
+
15357
+
15358
+
15359
+
15360
+
15361
+
15362
+
15363
+
15364
+
15365
+
15366
+
15367
+
15368
+
15369
+
15370
+
15371
+
15372
+
15373
+
15374
+
15375
+
15376
+
15377
+
15378
+
15379
+
15380
+
15381
+
15382
+
15383
+
15384
+
15385
+
15386
+
15387
+
15388
+
15389
+
15390
+
15391
+
15392
+
15393
+
15394
+
15395
+
15396
+
15397
+
15398
+
15399
+
15400
+
15401
+
15402
+
15403
+
15404
+
15405
+
15406
+
15407
+
15408
+
15409
+ Training...: 1%|██▎ | 750/61250 [05:48<8:09:04, 2.06it/s]
15410
+
15411
+
15412
+
15413
+
15414
+
15415
+
15416
+
15417
+
15418
+
15419
+
15420
+
15421
+
15422
+
15423
+
15424
+
15425
+
15426
+
15427
+
15428
+
15429
+
15430
+
15431
+
15432
+
15433
+
15434
+
15435
+
15436
+
15437
+
15438
+
15439
+
15440
+
15441
+
15442
+
15443
+
15444
+
15445
+
15446
+
15447
+
15448
+
15449
+
15450
+
15451
+
15452
+
15453
+
15454
+
15455
+
15456
+
15457
+
15458
+
15459
+
15460
+
15461
+
15462
+
15463
+
15464
+
15465
+
15466
+
15467
+
15468
+
15469
+
15470
+
15471
+
15472
+
15473
+
15474
+
15475
+
15476
+
15477
+
15478
+
15479
+
15480
+
15481
+
15482
+
15483
+
15484
+
15485
+
15486
+
15487
+
15488
+
15489
+
15490
+
15491
+
15492
+
15493
+
15494
+
15495
+
15496
+
15497
+
15498
+
15499
+
15500
+
15501
+
15502
+
15503
+
15504
+
15505
+
15506
+
15507
+
15508
+
15509
+
15510
+
15511
+
15512
+
15513
+
15514
+
15515
+
15516
+
15517
+
15518
+
15519
+
15520
+
15521
+
15522
+
15523
+ Training...: 2%|███▊ | 1250/61250 [09:51<8:05:02, 2.06it/s]
15524
+
15525
+
15526
+
15527
+
15528
+
15529
+
15530
+
15531
+
15532
+
15533
+
15534
+
15535
+
15536
+
15537
+
15538
+
15539
+
15540
+
15541
+
15542
+
15543
+
15544
+
15545
+
15546
+
15547
+
15548
+
15549
+
15550
+
15551
+
15552
+
15553
+
15554
+
15555
+
15556
+
15557
+
15558
+
15559
+
15560
+
15561
+
15562
+
15563
+
15564
+
15565
+
15566
+
15567
+
15568
+
15569
+
15570
+
15571
+
15572
+
15573
+
15574
+
15575
+
15576
+
15577
+
15578
+
15579
+
15580
+
15581
+
15582
+
15583
+
15584
+
15585
+
15586
+
15587
+
15588
+
15589
+
15590
+
15591
+
15592
+
15593
+
15594
+
15595
+
15596
+
15597
+
15598
+
15599
+
15600
+
15601
+
15602
+
15603
+
15604
+
15605
+
15606
+
15607
+
15608
+
15609
+
15610
+
15611
+
15612
+
15613
+
15614
+
15615
+
15616
+
15617
+
15618
+
15619
+
15620
+
15621
+
15622
+
15623
+
15624
+
15625
+
15626
+
15627
+
15628
+
15629
+
15630
+
15631
+
15632
+
15633
+
15634
+
15635
+
15636
+
15637
+ Training...: 3%|█████▎ | 1750/61250 [13:55<8:01:25, 2.06it/s]
15638
+
15639
+
15640
+
15641
+
15642
+
15643
+
15644
+
15645
+
15646
+
15647
+
15648
+
15649
+
15650
+
15651
+
15652
+
15653
+
15654
+
15655
+
15656
+
15657
+
15658
+
15659
+
15660
+
15661
+
15662
+
15663
+
15664
+
15665
+
15666
+
15667
+
15668
+
15669
+
15670
+
15671
+
15672
+
15673
+
15674
+
15675
+
15676
+
15677
+
15678
+
15679
+
15680
+
15681
+
15682
+
15683
+
15684
+
15685
+
15686
+
15687
+
15688
+
15689
+
15690
+
15691
+
15692
+
15693
+
15694
+
15695
+
15696
+
15697
+
15698
+
15699
+
15700
+
15701
+
15702
+
15703
+
15704
+
15705
+
15706
+
15707
+
15708
+
15709
+
15710
+
15711
+
15712
+
15713
+
15714
+
15715
+
15716
+
15717
+
15718
+
15719
+
15720
+
15721
+
15722
+
15723
+
15724
+
15725
+
15726
+
15727
+
15728
+
15729
+
15730
+
15731
+
15732
+
15733
+
15734
+
15735
+
15736
+
15737
+
15738
+
15739
+
15740
+
15741
+
15742
+
15743
+
15744
+
15745
+
15746
+
15747
+
15748
+
15749
+
15750
+
15751
+
15752
+ Training...: 4%|██████▉ | 2277/61250 [18:16<1:27:14, 11.27it/s]
15753
+
15754
+
15755
+
15756
+
15757
+
15758
+
15759
+
15760
+
15761
+
15762
+
15763
+
15764
+
15765
+
15766
+
15767
+
15768
+
15769
+
15770
+
15771
+
15772
+
15773
+
15774
+
15775
+
15776
+
15777
+
15778
+
15779
+
15780
+
15781
+
15782
+
15783
+
15784
+
15785
+
15786
+
15787
+
15788
+
15789
+
15790
+
15791
+
15792
+
15793
+
15794
+
15795
+
15796
+
15797
+
15798
+
15799
+
15800
+
15801
+
15802
+
15803
+
15804
+
15805
+
15806
+
15807
+
15808
+
15809
+
15810
+
15811
+
15812
+
15813
+
15814
+
15815
+
15816
+
15817
+
15818
+
15819
+
15820
+
15821
+
15822
+
15823
+
15824
+
15825
+
15826
+
15827
+
15828
+
15829
+
15830
+
15831
+
15832
+
15833
+
15834
+
15835
+
15836
+
15837
+
15838
+
15839
+
15840
+
15841
+
15842
+
15843
+
15844
+
15845
+
15846
+
15847
+
15848
+
15849
+
15850
+
15851
+
15852
+
15853
+
15854
+
15855
+
15856
+
15857
+
15858
+
15859
+
15860
+
15861
+
15862
+
15863
+
15864
+
15865
+ Training...: 4%|████████▍ | 2750/61250 [22:01<7:52:56, 2.06it/s]
15866
+
15867
+
15868
+
15869
+
15870
+
15871
+
15872
+
15873
+
15874
+
15875
+
15876
+
15877
+
15878
+
15879
+
15880
+
15881
+
15882
+
15883
+
15884
+
15885
+
15886
+
15887
+
15888
+
15889
+
15890
+
15891
+
15892
+
15893
+
15894
+
15895
+
15896
+
15897
+
15898
+
15899
+
15900
+
15901
+
15902
+
15903
+
15904
+
15905
+
15906
+
15907
+
15908
+
15909
+
15910
+
15911
+
15912
+
15913
+
15914
+
15915
+
15916
+
15917
+
15918
+
15919
+
15920
+
15921
+
15922
+
15923
+
15924
+
15925
+
15926
+
15927
+
15928
+
15929
+
15930
+
15931
+
15932
+
15933
+
15934
+
15935
+
15936
+
15937
+
15938
+
15939
+
15940
+
15941
+
15942
+
15943
+
15944
+
15945
+
15946
+
15947
+
15948
+
15949
+
15950
+
15951
+
15952
+
15953
+
15954
+
15955
+
15956
+
15957
+
15958
+
15959
+
15960
+
15961
+
15962
+
15963
+
15964
+
15965
+
15966
+
15967
+
15968
+
15969
+
15970
+
15971
+
15972
+
15973
+
15974
+
15975
+
15976
+
15977
+
15978
+
15979
+
15980
+ Training...: 5%|█████████▉ | 3250/61250 [26:04<7:48:48, 2.06it/s]
15981
+
15982
+
15983
+
15984
+
15985
+
15986
+
15987
+
15988
+
15989
+
15990
+
15991
+
15992
+
15993
+
15994
+
15995
+
15996
+
15997
+
15998
+
15999
+
16000
+
16001
+
16002
+
16003
+
16004
+
16005
+
16006
+
16007
+
16008
+
16009
+
16010
+
16011
+
16012
+
16013
+
16014
+
16015
+
16016
+
16017
+
16018
+
16019
+
16020
+
16021
+
16022
+
16023
+
16024
+
16025
+
16026
+
16027
+
16028
+
16029
+
16030
+
16031
+
16032
+
16033
+
16034
+
16035
+
16036
+
16037
+
16038
+
16039
+
16040
+
16041
+
16042
+
16043
+
16044
+
16045
+
16046
+
16047
+
16048
+
16049
+
16050
+
16051
+
16052
+
16053
+
16054
+
16055
+
16056
+
16057
+
16058
+
16059
+
16060
+
16061
+
16062
+
16063
+
16064
+
16065
+
16066
+
16067
+
16068
+
16069
+
16070
+
16071
+
16072
+
16073
+
16074
+
16075
+
16076
+
16077
+
16078
+
16079
+
16080
+
16081
+
16082
+
16083
+
16084
+
16085
+
16086
+
16087
+
16088
+
16089
+
16090
+
16091
+
16092
+
16093
+
16094
+
16095
+ Training...: 6%|███████████▌ | 3750/61250 [30:24<7:44:42, 2.06it/s]
16096
+ Evaluating ...: 0%|▌ | 2/625 [00:00<01:07, 9.25it/s]
16097
+
16098
+
16099
+
16100
+
16101
+
16102
+
16103
+
16104
+
16105
+
16106
+
16107
+
16108
+
16109
+
16110
+
16111
+
16112
+
16113
+
16114
+
16115
+
16116
+
16117
+
16118
+
16119
+
16120
+
16121
+
16122
+
16123
+
16124
+
16125
+
16126
+
16127
+
16128
+
16129
+
16130
+
16131
+
16132
+
16133
+
16134
+
16135
+
16136
+
16137
+
16138
+
16139
+
16140
+
16141
+
16142
+
16143
+
16144
+
16145
+
16146
+
16147
+
16148
+
16149
+
16150
+
16151
+
16152
+
16153
+
16154
+
16155
+
16156
+
16157
+
16158
+
16159
+
16160
+
16161
+
16162
+
16163
+
16164
+
16165
+
16166
+
16167
+
16168
+
16169
+
16170
+
16171
+
16172
+
16173
+
16174
+
16175
+
16176
+
16177
+
16178
+
16179
+
16180
+
16181
+
16182
+
16183
+
16184
+
16185
+
16186
+
16187
+
16188
+
16189
+
16190
+
16191
+
16192
+
16193
+
16194
+
16195
+
16196
+
16197
+
16198
+
16199
+
16200
+
16201
+
16202
+
16203
+
16204
+
16205
+
16206
+
16207
+
16208
+
16209
+
16210
+
16211
+
16212
+
16213
+
16214
+
16215
+
16216
+
16217
+
16218
+
16219
+
16220
+
16221
+
16222
+
16223
+
16224
+
16225
+
16226
+
16227
+
16228
+
16229
+
16230
+
16231
+
16232
+
16233
+
16234
+
16235
+
16236
+
16237
+
16238
+
16239
+
16240
+
16241
+
16242
+
16243
+
16244
+
16245
+
16246
+
16247
+
16248
+
16249
+
16250
+
16251
+
16252
+
16253
+
16254
+
16255
+
16256
+
16257
+
16258
+
16259
+
16260
+
16261
+
16262
+
16263
+
16264
+
16265
+
16266
+
16267
+
16268
+
16269
+
16270
+
16271
+
16272
+
16273
+
16274
+
16275
+
16276
+
16277
+
16278
+
16279
+
16280
+
16281
+
16282
+
16283
+
16284
+
16285
+
16286
+
16287
+
16288
+
16289
+
16290
+
16291
+
16292
+
16293
+
16294
+
16295
+
16296
+
16297
+
16298
+
16299
+ Training...: 7%|█████████████ | 4250/61250 [37:18<7:40:45, 2.06it/s]
16300
+
16301
+
16302
+
16303
+
16304
+
16305
+
16306
+
16307
+
16308
+
16309
+
16310
+
16311
+
16312
+
16313
+
16314
+
16315
+
16316
+
16317
+
16318
+
16319
+
16320
+
16321
+
16322
+
16323
+
16324
+
16325
+
16326
+
16327
+
16328
+
16329
+
16330
+
16331
+
16332
+
16333
+
16334
+
16335
+
16336
+
16337
+
16338
+
16339
+
16340
+
16341
+
16342
+
16343
+
16344
+
16345
+
16346
+
16347
+
16348
+
16349
+
16350
+
16351
+
16352
+
16353
+
16354
+
16355
+
16356
+
16357
+
16358
+
16359
+
16360
+
16361
+
16362
+
16363
+
16364
+
16365
+
16366
+
16367
+
16368
+
16369
+
16370
+
16371
+
16372
+
16373
+
16374
+
16375
+
16376
+
16377
+
16378
+
16379
+
16380
+
16381
+
16382
+
16383
+
16384
+
16385
+
16386
+
16387
+
16388
+
16389
+
16390
+
16391
+
16392
+
16393
+
16394
+
16395
+
16396
+
16397
+
16398
+
16399
+
16400
+
16401
+
16402
+
16403
+
16404
+
16405
+
16406
+
16407
+
16408
+
16409
+
16410
+
16411
+
16412
+
16413
+
16414
+ Training...: 8%|██████████████▌ | 4750/61250 [41:21<7:36:38, 2.06it/s]
16415
+
16416
+
16417
+
16418
+
16419
+
16420
+
16421
+
16422
+
16423
+
16424
+
16425
+
16426
+
16427
+
16428
+
16429
+
16430
+
16431
+
16432
+
16433
+
16434
+
16435
+
16436
+
16437
+
16438
+
16439
+
16440
+
16441
+
16442
+
16443
+
16444
+
16445
+
16446
+
16447
+
16448
+
16449
+
16450
+
16451
+
16452
+
16453
+
16454
+
16455
+
16456
+
16457
+
16458
+
16459
+
16460
+
16461
+
16462
+
16463
+
16464
+
16465
+
16466
+
16467
+
16468
+
16469
+
16470
+
16471
+
16472
+
16473
+
16474
+
16475
+
16476
+
16477
+
16478
+
16479
+
16480
+
16481
+
16482
+
16483
+
16484
+
16485
+
16486
+
16487
+
16488
+
16489
+
16490
+
16491
+
16492
+
16493
+
16494
+
16495
+
16496
+
16497
+
16498
+
16499
+
16500
+
16501
+
16502
+
16503
+
16504
+
16505
+
16506
+
16507
+
16508
+
16509
+
16510
+
16511
+
16512
+
16513
+
16514
+
16515
+
16516
+
16517
+
16518
+
16519
+
16520
+
16521
+
16522
+
16523
+
16524
+
16525
+
16526
+
16527
+
16528
+
16529
+
16530
+ Training...: 9%|████████████████ | 5257/61250 [45:41<17:51:54, 1.15s/it]
16531
+
16532
+
16533
+
16534
+
16535
+
16536
+
16537
+
16538
+
16539
+
16540
+
16541
+
16542
+
16543
+
16544
+
16545
+
16546
+
16547
+
16548
+
16549
+
16550
+
16551
+
16552
+
16553
+
16554
+
16555
+
16556
+
16557
+
16558
+
16559
+
16560
+
16561
+
16562
+
16563
+
16564
+
16565
+
16566
+
16567
+
16568
+
16569
+
16570
+
16571
+
16572
+
16573
+
16574
+
16575
+
16576
+
16577
+
16578
+
16579
+
16580
+
16581
+
16582
+
16583
+
16584
+
16585
+
16586
+
16587
+
16588
+
16589
+
16590
+
16591
+
16592
+
16593
+
16594
+
16595
+
16596
+
16597
+
16598
+
16599
+
16600
+
16601
+
16602
+
16603
+
16604
+
16605
+
16606
+
16607
+
16608
+
16609
+
16610
+
16611
+
16612
+
16613
+
16614
+
16615
+
16616
+
16617
+
16618
+
16619
+
16620
+
16621
+
16622
+
16623
+
16624
+
16625
+
16626
+
16627
+
16628
+
16629
+
16630
+
16631
+
16632
+
16633
+
16634
+
16635
+
16636
+
16637
+
16638
+
16639
+
16640
+
16641
+
16642
+
16643
+
16644
+
16645
+ Training...: 9%|█████████████████▌ | 5757/61250 [49:44<17:42:34, 1.15s/it]
16646
+
16647
+
16648
+
16649
+
16650
+
16651
+
16652
+
16653
+
16654
+
16655
+
16656
+
16657
+
16658
+
16659
+
16660
+
16661
+
16662
+
16663
+
16664
+
16665
+
16666
+
16667
+
16668
+
16669
+
16670
+
16671
+
16672
+
16673
+
16674
+
16675
+
16676
+
16677
+
16678
+
16679
+
16680
+
16681
+
16682
+
16683
+
16684
+
16685
+
16686
+
16687
+
16688
+
16689
+
16690
+
16691
+
16692
+
16693
+
16694
+
16695
+
16696
+
16697
+
16698
+
16699
+
16700
+
16701
+
16702
+
16703
+
16704
+
16705
+
16706
+
16707
+
16708
+
16709
+
16710
+
16711
+
16712
+
16713
+
16714
+
16715
+
16716
+
16717
+
16718
+
16719
+
16720
+
16721
+
16722
+
16723
+
16724
+
16725
+
16726
+
16727
+
16728
+
16729
+
16730
+
16731
+
16732
+
16733
+
16734
+
16735
+
16736
+
16737
+
16738
+
16739
+
16740
+
16741
+
16742
+
16743
+
16744
+
16745
+
16746
+
16747
+
16748
+
16749
+
16750
+
16751
+
16752
+
16753
+
16754
+
16755
+
16756
+
16757
+
16758
+
16759
+
16760
+ Training...: 10%|███████████████████ | 6255/61250 [53:47<26:57:17, 1.76s/it]
16761
+
16762
+
16763
+
16764
+
16765
+
16766
+
16767
+
16768
+
16769
+
16770
+
16771
+
16772
+
16773
+
16774
+
16775
+
16776
+
16777
+
16778
+
16779
+
16780
+
16781
+
16782
+
16783
+
16784
+
16785
+
16786
+
16787
+
16788
+
16789
+
16790
+
16791
+
16792
+
16793
+
16794
+
16795
+
16796
+
16797
+
16798
+
16799
+
16800
+
16801
+
16802
+
16803
+
16804
+
16805
+
16806
+
16807
+
16808
+
16809
+
16810
+
16811
+
16812
+
16813
+
16814
+
16815
+
16816
+
16817
+
16818
+
16819
+
16820
+
16821
+
16822
+
16823
+
16824
+
16825
+
16826
+
16827
+
16828
+
16829
+
16830
+
16831
+
16832
+
16833
+
16834
+
16835
+
16836
+
16837
+
16838
+
16839
+
16840
+
16841
+
16842
+
16843
+
16844
+
16845
+
16846
+
16847
+
16848
+
16849
+
16850
+
16851
+
16852
+
16853
+
16854
+
16855
+
16856
+
16857
+
16858
+
16859
+
16860
+
16861
+
16862
+
16863
+
16864
+
16865
+
16866
+
16867
+
16868
+
16869
+
16870
+
16871
+
16872
+
16873
+
16874
+
16875
+ Training...: 11%|████████████████████▌ | 6755/61250 [57:51<26:42:00, 1.76s/it]
16876
+
16877
+
16878
+
16879
+
16880
+
16881
+
16882
+
16883
+
16884
+
16885
+
16886
+
16887
+
16888
+
16889
+
16890
+
16891
+
16892
+
16893
+
16894
+
16895
+
16896
+
16897
+
16898
+
16899
+
16900
+
16901
+
16902
+
16903
+
16904
+
16905
+
16906
+
16907
+
16908
+
16909
+
16910
+
16911
+
16912
+
16913
+
16914
+
16915
+
16916
+
16917
+
16918
+
16919
+
16920
+
16921
+
16922
+
16923
+
16924
+
16925
+
16926
+
16927
+
16928
+
16929
+
16930
+
16931
+
16932
+
16933
+
16934
+
16935
+
16936
+
16937
+
16938
+
16939
+
16940
+
16941
+
16942
+
16943
+
16944
+
16945
+
16946
+
16947
+
16948
+
16949
+
16950
+
16951
+
16952
+
16953
+
16954
+
16955
+
16956
+
16957
+
16958
+
16959
+
16960
+
16961
+
16962
+
16963
+
16964
+
16965
+
16966
+
16967
+
16968
+
16969
+
16970
+
16971
+
16972
+
16973
+
16974
+
16975
+
16976
+
16977
+
16978
+
16979
+
16980
+
16981
+
16982
+
16983
+
16984
+
16985
+
16986
+
16987
+
16988
+
16989
+
16990
+ Training...: 12%|█████████████████████▉ | 7253/61250 [1:01:54<43:14:07, 2.88s/it]
16991
+
16992
+
16993
+
16994
+
16995
+
16996
+
16997
+
16998
+
16999
+
17000
+
17001
+
17002
+
17003
+
17004
+
17005
+
17006
+
17007
+
17008
+
17009
+
17010
+
17011
+
17012
+
17013
+
17014
+
17015
+
17016
+
17017
+
17018
+
17019
+
17020
+
17021
+
17022
+
17023
+
17024
+
17025
+
17026
+
17027
+
17028
+
17029
+
17030
+
17031
+
17032
+
17033
+
17034
+
17035
+
17036
+
17037
+
17038
+
17039
+
17040
+
17041
+
17042
+
17043
+
17044
+
17045
+
17046
+
17047
+
17048
+
17049
+
17050
+
17051
+
17052
+
17053
+
17054
+
17055
+
17056
+
17057
+
17058
+
17059
+
17060
+
17061
+
17062
+
17063
+
17064
+
17065
+
17066
+
17067
+
17068
+
17069
+
17070
+
17071
+
17072
+
17073
+
17074
+
17075
+
17076
+
17077
+
17078
+
17079
+
17080
+
17081
+
17082
+
17083
+
17084
+
17085
+
17086
+
17087
+
17088
+
17089
+
17090
+
17091
+
17092
+
17093
+
17094
+
17095
+
17096
+
17097
+
17098
+
17099
+
17100
+
17101
+
17102
+
17103
+
17104
+
17105
+ Training...: 13%|███████████████████████▍ | 7753/61250 [1:05:57<42:53:26, 2.89s/it]
17106
+
17107
+
17108
+
17109
+
17110
+
17111
+
17112
+
17113
+
17114
+
17115
+
17116
+
17117
+
17118
+
17119
+
17120
+
17121
+
17122
+
17123
+
17124
+
17125
+
17126
+
17127
+
17128
+
17129
+
17130
+
17131
+
17132
+
17133
+
17134
+
17135
+
17136
+
17137
+
17138
+
17139
+
17140
+
17141
+
17142
+
17143
+
17144
+
17145
+
17146
+
17147
+
17148
+
17149
+
17150
+
17151
+
17152
+
17153
+
17154
+
17155
+
17156
+
17157
+
17158
+
17159
+
17160
+
17161
+
17162
+
17163
+
17164
+
17165
+
17166
+
17167
+
17168
+
17169
+
17170
+
17171
+
17172
+
17173
+
17174
+
17175
+
17176
+
17177
+
17178
+
17179
+
17180
+
17181
+
17182
+
17183
+
17184
+
17185
+
17186
+
17187
+
17188
+
17189
+
17190
+
17191
+
17192
+
17193
+
17194
+
17195
+
17196
+
17197
+
17198
+
17199
+
17200
+
17201
+
17202
+
17203
+
17204
+
17205
+
17206
+
17207
+
17208
+
17209
+
17210
+
17211
+
17212
+
17213
+
17214
+
17215
+
17216
+
17217
+
17218
+
17219
+
17220
+ Training...: 13%|████████████████████████▉ | 8253/61250 [1:10:00<42:28:41, 2.89s/it]
17221
+
17222
+
17223
+
17224
+
17225
+
17226
+
17227
+
17228
+
17229
+
17230
+
17231
+
17232
+
17233
+
17234
+
17235
+
17236
+
17237
+
17238
+
17239
+
17240
+
17241
+
17242
+
17243
+
17244
+
17245
+
17246
+
17247
+
17248
+
17249
+
17250
+
17251
+
17252
+
17253
+
17254
+
17255
+
17256
+
17257
+
17258
+
17259
+
17260
+
17261
+
17262
+
17263
+
17264
+
17265
+
17266
+
17267
+
17268
+
17269
+
17270
+
17271
+
17272
+
17273
+
17274
+
17275
+
17276
+
17277
+
17278
+
17279
+
17280
+
17281
+
17282
+
17283
+
17284
+
17285
+
17286
+
17287
+
17288
+
17289
+
17290
+
17291
+
17292
+
17293
+
17294
+
17295
+
17296
+
17297
+
17298
+
17299
+
17300
+
17301
+
17302
+
17303
+
17304
+
17305
+
17306
+
17307
+
17308
+
17309
+
17310
+
17311
+
17312
+
17313
+
17314
+
17315
+
17316
+
17317
+
17318
+
17319
+
17320
+
17321
+
17322
+
17323
+
17324
+
17325
+
17326
+
17327
+
17328
+
17329
+
17330
+
17331
+
17332
+
17333
+
17334
+ Training...: 14%|██████████████████████████▌ | 8750/61250 [1:13:46<7:04:26, 2.06it/s]
17335
+ Training...: 14%|██████████████████████████▌ | 8750/61250 [1:14:03<7:04:26, 2.06it/s]
17336
+
17337
+
17338
+
17339
+
17340
+
17341
+
17342
+
17343
+
17344
+
17345
+
17346
+
17347
+
17348
+
17349
+
17350
+
17351
+
17352
+
17353
+
17354
+
17355
+
17356
+
17357
+
17358
+
17359
+
17360
+
17361
+
17362
+
17363
+
17364
+
17365
+
17366
+
17367
+
17368
+
17369
+
17370
+
17371
+
17372
+
17373
+
17374
+
17375
+
17376
+
17377
+
17378
+
17379
+
17380
+
17381
+
17382
+
17383
+
17384
+
17385
+
17386
+
17387
+
17388
+
17389
+
17390
+
17391
+
17392
+
17393
+
17394
+
17395
+
17396
+
17397
+
17398
+
17399
+
17400
+
17401
+
17402
+
17403
+
17404
+
17405
+
17406
+
17407
+
17408
+
17409
+
17410
+
17411
+
17412
+
17413
+
17414
+
17415
+
17416
+
17417
+
17418
+
17419
+
17420
+
17421
+
17422
+
17423
+
17424
+
17425
+
17426
+
17427
+
17428
+
17429
+
17430
+
17431
+
17432
+
17433
+
17434
+
17435
+
17436
+
17437
+
17438
+
17439
+
17440
+
17441
+
17442
+
17443
+
17444
+
17445
+
17446
+
17447
+
17448
+
17449
+
17450
+
17451
+
17452
+
17453
+
17454
+
17455
+
17456
+
17457
+
17458
+
17459
+
17460
+
17461
+
17462
+
17463
+
17464
+
17465
+
17466
+
17467
+
17468
+
17469
+
17470
+
17471
+
17472
+
17473
+
17474
+
17475
+
17476
+
17477
+
17478
+
17479
+
17480
+
17481
+
17482
+
17483
+
17484
+
17485
+
17486
+
17487
+
17488
+
17489
+
17490
+
17491
+
17492
+
17493
+
17494
+
17495
+
17496
+
17497
+
17498
+
17499
+
17500
+
17501
+
17502
+
17503
+
17504
+
17505
+
17506
+
17507
+
17508
+
17509
+
17510
+
17511
+
17512
+
17513
+
17514
+
17515
+
17516
+
17517
+
17518
+
17519
+
17520
+
17521
+
17522
+
17523
+
17524
+
17525
+
17526
+
17527
+
17528
+
17529
+
17530
+
17531
+
17532
+
17533
+
17534
+
17535
+
17536
+
17537
+ Training...: 15%|████████████████████████████ | 9250/61250 [1:20:57<7:00:16, 2.06it/s]
17538
+
17539
+
17540
+
17541
+
17542
+
17543
+
17544
+
17545
+
17546
+
17547
+
17548
+
17549
+
17550
+
17551
+
17552
+
17553
+
17554
+
17555
+
17556
+
17557
+
17558
+
17559
+
17560
+
17561
+
17562
+
17563
+
17564
+
17565
+
17566
+
17567
+
17568
+
17569
+
17570
+
17571
+
17572
+
17573
+
17574
+
17575
+
17576
+
17577
+
17578
+
17579
+
17580
+
17581
+
17582
+
17583
+
17584
+
17585
+
17586
+
17587
+
17588
+
17589
+
17590
+
17591
+
17592
+
17593
+
17594
+
17595
+
17596
+
17597
+
17598
+
17599
+
17600
+
17601
+
17602
+
17603
+
17604
+
17605
+
17606
+
17607
+
17608
+
17609
+
17610
+
17611
+
17612
+
17613
+
17614
+
17615
+
17616
+
17617
+
17618
+
17619
+
17620
+
17621
+
17622
+
17623
+
17624
+
17625
+
17626
+
17627
+
17628
+
17629
+
17630
+
17631
+
17632
+
17633
+
17634
+
17635
+
17636
+
17637
+
17638
+
17639
+
17640
+
17641
+
17642
+
17643
+
17644
+
17645
+
17646
+
17647
+
17648
+
17649
+
17650
+
17651
+ Training...: 16%|█████████████████████████████▌ | 9750/61250 [1:25:00<6:56:21, 2.06it/s]
17652
+
17653
+
17654
+
17655
+
17656
+
17657
+
17658
+
17659
+
17660
+
17661
+
17662
+
17663
+
17664
+
17665
+
17666
+
17667
+
17668
+
17669
+
17670
+
17671
+
17672
+
17673
+
17674
+
17675
+
17676
+
17677
+
17678
+
17679
+
17680
+
17681
+
17682
+
17683
+
17684
+
17685
+
17686
+
17687
+
17688
+
17689
+
17690
+
17691
+
17692
+
17693
+
17694
+
17695
+
17696
+
17697
+
17698
+
17699
+
17700
+
17701
+
17702
+
17703
+
17704
+
17705
+
17706
+
17707
+
17708
+
17709
+
17710
+
17711
+
17712
+
17713
+
17714
+
17715
+
17716
+
17717
+
17718
+
17719
+
17720
+
17721
+
17722
+
17723
+
17724
+
17725
+
17726
+
17727
+
17728
+
17729
+
17730
+
17731
+
17732
+
17733
+
17734
+
17735
+
17736
+
17737
+
17738
+
17739
+
17740
+
17741
+
17742
+
17743
+
17744
+
17745
+
17746
+
17747
+
17748
+
17749
+
17750
+
17751
+
17752
+
17753
+
17754
+
17755
+
17756
+
17757
+
17758
+
17759
+
17760
+
17761
+
17762
+
17763
+
17764
+
17765
+
17766
+ Training...: 17%|██████████████████████████████▉ | 10250/61250 [1:29:04<6:52:18, 2.06it/s]
17767
+
17768
+
17769
+
17770
+
17771
+
17772
+
17773
+
17774
+
17775
+
17776
+
17777
+
17778
+
17779
+
17780
+
17781
+
17782
+
17783
+
17784
+
17785
+
17786
+
17787
+
17788
+
17789
+
17790
+
17791
+
17792
+
17793
+
17794
+
17795
+
17796
+
17797
+
17798
+
17799
+
17800
+
17801
+
17802
+
17803
+
17804
+
17805
+
17806
+
17807
+
17808
+
17809
+
17810
+
17811
+
17812
+
17813
+
17814
+
17815
+
17816
+
17817
+
17818
+
17819
+
17820
+
17821
+
17822
+
17823
+
17824
+
17825
+
17826
+
17827
+
17828
+
17829
+
17830
+
17831
+
17832
+
17833
+
17834
+
17835
+
17836
+
17837
+
17838
+
17839
+
17840
+
17841
+
17842
+
17843
+
17844
+
17845
+
17846
+
17847
+
17848
+
17849
+
17850
+
17851
+
17852
+
17853
+
17854
+
17855
+
17856
+
17857
+
17858
+
17859
+
17860
+
17861
+
17862
+
17863
+
17864
+
17865
+
17866
+
17867
+
17868
+
17869
+
17870
+
17871
+
17872
+
17873
+
17874
+
17875
+
17876
+
17877
+
17878
+
17879
+
17880
+ Step... (70000 | Loss: 1.39120352268219, Acc: 0.7222487330436707): 33%|████████████████████████████████████████████▋ | 1/3 [10:38:19<21:16:38, 38299.27s/it]
17881
+ Traceback (most recent call last):
17882
+ File "run_muril_bb_flax_mlm.py", line 709, in <module>
17883
+ write_train_metric(summary_writer, train_metrics, train_time, cur_step)
17884
+ File "run_muril_bb_flax_mlm.py", line 269, in write_train_metric
17885
+ train_metrics = get_metrics(train_metrics)
17886
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/training/common_utils.py", line 52, in get_metrics
17887
+ metrics_np = jax.device_get(device_metrics)
17888
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/_src/api.py", line 2682, in device_get
17889
+ return tree_map(_device_get, x)
17890
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/_src/tree_util.py", line 178, in tree_map
17891
+ return treedef.unflatten(f(*xs) for xs in zip(*all_leaves))
17892
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/_src/tree_util.py", line 178, in <genexpr>
17893
+ return treedef.unflatten(f(*xs) for xs in zip(*all_leaves))
17894
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/_src/api.py", line 2646, in _device_get
17895
+ return copy()
17896
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/interpreters/xla.py", line 1428, in copy
17897
+ return np.asarray(self)
17898
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/interpreters/xla.py", line 1496, in __array__
17899
+ return np.asarray(self._value, dtype=dtype)
17900
+ KeyboardInterrupt
wandb/run-20211107_204808-2ns3lo5n/files/wandb-summary.json CHANGED
@@ -1 +1 @@
1
- {"training_step": 100000, "learning_rate": 3.639455826487392e-05, "train_loss": 1.2350531816482544, "_runtime": 26755, "_timestamp": 1636344843, "_step": 111, "eval_step": 100000, "eval_accuracy": 0.7188425660133362, "eval_loss": 1.41820228099823}
 
1
+ {"training_step": 71500, "learning_rate": 3.054421904380433e-05, "train_loss": 1.3594125509262085, "_runtime": 38088, "_timestamp": 1636356176, "_step": 158, "eval_step": 70000, "eval_accuracy": 0.7222487330436707, "eval_loss": 1.39120352268219, "_wandb": {"runtime": 38320}}
wandb/run-20211107_204808-2ns3lo5n/logs/debug-internal.log CHANGED
The diff for this file is too large to render. See raw diff
 
wandb/run-20211107_204808-2ns3lo5n/logs/debug.log CHANGED
@@ -23,3 +23,125 @@ config: {}
23
  2021-11-07 20:48:11,628 INFO MainThread:20231 [wandb_run.py:_config_callback():911] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': True, 'do_eval': True, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 2, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 5e-05, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 3.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 0, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Nov07_20-47-12_t1v-n-ba7aa1a5-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 500, 'logging_nan_inf_filter': True, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 50000, 'save_total_limit': 1, 'save_on_each_node': False, 'no_cuda': False, 'seed': 18, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'xpu_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 5000, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': 'en-ta-hi-te-from-50k', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'HubStrategy.EVERY_SAVE', 'hub_token': None, 'gradient_checkpointing': False, 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': '', '_n_gpu': 0, '__cached__setup_devices': 'cpu'}
24
  2021-11-07 20:48:11,630 INFO MainThread:20231 [wandb_run.py:_config_callback():911] config_cb None None {'model_name_or_path': 'muril-bigbird', 'model_type': None, 'config_name': None, 'tokenizer_name': 'muril-bigbird', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'bfloat16'}
25
  2021-11-07 20:48:11,631 INFO MainThread:20231 [wandb_run.py:_config_callback():911] config_cb None None {'dataset_name': None, 'pretokenized': True, 'streaming': False, 'dataset_config_name': None, 'train_file': '/dev/shm/data-files', 'validation_file': '.', 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': None, 'mlm_probability': 0.15, 'pad_to_max_length': True, 'line_by_line': False}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
23
  2021-11-07 20:48:11,628 INFO MainThread:20231 [wandb_run.py:_config_callback():911] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': True, 'do_eval': True, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 2, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 5e-05, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 3.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 0, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Nov07_20-47-12_t1v-n-ba7aa1a5-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 500, 'logging_nan_inf_filter': True, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 50000, 'save_total_limit': 1, 'save_on_each_node': False, 'no_cuda': False, 'seed': 18, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'xpu_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 5000, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': 'en-ta-hi-te-from-50k', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'HubStrategy.EVERY_SAVE', 'hub_token': None, 'gradient_checkpointing': False, 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': '', '_n_gpu': 0, '__cached__setup_devices': 'cpu'}
24
  2021-11-07 20:48:11,630 INFO MainThread:20231 [wandb_run.py:_config_callback():911] config_cb None None {'model_name_or_path': 'muril-bigbird', 'model_type': None, 'config_name': None, 'tokenizer_name': 'muril-bigbird', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'bfloat16'}
25
  2021-11-07 20:48:11,631 INFO MainThread:20231 [wandb_run.py:_config_callback():911] config_cb None None {'dataset_name': None, 'pretokenized': True, 'streaming': False, 'dataset_config_name': None, 'train_file': '/dev/shm/data-files', 'validation_file': '.', 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': None, 'mlm_probability': 0.15, 'pad_to_max_length': True, 'line_by_line': False}
26
+ 2021-11-08 07:26:47,472 INFO MainThread:20231 [wandb_run.py:_atexit_cleanup():1663] got exitcode: 255
27
+ 2021-11-08 07:26:47,472 INFO MainThread:20231 [wandb_run.py:_restore():1635] restore
28
+ 2021-11-08 07:26:50,100 INFO MainThread:20231 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
29
+ wandb_count: 1
30
+ }
31
+ pusher_stats {
32
+ uploaded_bytes: 1758
33
+ total_bytes: 1758
34
+ }
35
+
36
+ 2021-11-08 07:26:50,366 INFO MainThread:20231 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
37
+ wandb_count: 1
38
+ }
39
+ pusher_stats {
40
+ uploaded_bytes: 1758
41
+ total_bytes: 1758
42
+ }
43
+
44
+ 2021-11-08 07:26:50,710 INFO MainThread:20231 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
45
+ wandb_count: 5
46
+ }
47
+ pusher_stats {
48
+ uploaded_bytes: 1758
49
+ total_bytes: 95883
50
+ }
51
+
52
+ 2021-11-08 07:26:50,812 INFO MainThread:20231 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
53
+ wandb_count: 5
54
+ }
55
+ pusher_stats {
56
+ uploaded_bytes: 1758
57
+ total_bytes: 95883
58
+ }
59
+
60
+ 2021-11-08 07:26:50,914 INFO MainThread:20231 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
61
+ wandb_count: 5
62
+ }
63
+ pusher_stats {
64
+ uploaded_bytes: 95883
65
+ total_bytes: 95883
66
+ }
67
+
68
+ 2021-11-08 07:26:51,016 INFO MainThread:20231 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
69
+ wandb_count: 5
70
+ }
71
+ pusher_stats {
72
+ uploaded_bytes: 95883
73
+ total_bytes: 95883
74
+ }
75
+
76
+ 2021-11-08 07:26:51,117 INFO MainThread:20231 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
77
+ wandb_count: 5
78
+ }
79
+ pusher_stats {
80
+ uploaded_bytes: 95883
81
+ total_bytes: 95883
82
+ }
83
+
84
+ 2021-11-08 07:26:51,219 INFO MainThread:20231 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
85
+ wandb_count: 5
86
+ }
87
+ pusher_stats {
88
+ uploaded_bytes: 95883
89
+ total_bytes: 95883
90
+ }
91
+
92
+ 2021-11-08 07:26:51,321 INFO MainThread:20231 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
93
+ wandb_count: 5
94
+ }
95
+ pusher_stats {
96
+ uploaded_bytes: 95883
97
+ total_bytes: 95883
98
+ }
99
+
100
+ 2021-11-08 07:26:51,423 INFO MainThread:20231 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
101
+ wandb_count: 5
102
+ }
103
+ pusher_stats {
104
+ uploaded_bytes: 95883
105
+ total_bytes: 95883
106
+ }
107
+
108
+ 2021-11-08 07:26:51,525 INFO MainThread:20231 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
109
+ wandb_count: 5
110
+ }
111
+ pusher_stats {
112
+ uploaded_bytes: 95883
113
+ total_bytes: 95883
114
+ }
115
+
116
+ 2021-11-08 07:26:51,626 INFO MainThread:20231 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
117
+ wandb_count: 5
118
+ }
119
+ pusher_stats {
120
+ uploaded_bytes: 95883
121
+ total_bytes: 95883
122
+ }
123
+
124
+ 2021-11-08 07:26:51,946 INFO MainThread:20231 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
125
+ wandb_count: 5
126
+ }
127
+ pusher_stats {
128
+ uploaded_bytes: 95883
129
+ total_bytes: 95883
130
+ }
131
+
132
+ 2021-11-08 07:26:52,296 INFO MainThread:20231 [wandb_run.py:_wait_for_finish():1793] got exit ret: done: true
133
+ exit_result {
134
+ }
135
+ file_counts {
136
+ wandb_count: 5
137
+ }
138
+ pusher_stats {
139
+ uploaded_bytes: 95883
140
+ total_bytes: 95883
141
+ }
142
+ local_info {
143
+ }
144
+
145
+ 2021-11-08 07:26:53,747 INFO MainThread:20231 [wandb_run.py:_append_history():2011] rendering history
146
+ 2021-11-08 07:26:53,748 INFO MainThread:20231 [wandb_run.py:_append_summary():1966] rendering summary
147
+ 2021-11-08 07:26:53,748 INFO MainThread:20231 [wandb_run.py:_append_files():2061] logging synced files
wandb/run-20211107_204808-2ns3lo5n/run-2ns3lo5n.wandb CHANGED
Binary files a/wandb/run-20211107_204808-2ns3lo5n/run-2ns3lo5n.wandb and b/wandb/run-20211107_204808-2ns3lo5n/run-2ns3lo5n.wandb differ
 
wandb/run-20211108_073656-hyqam6zi/files/config.yaml ADDED
@@ -0,0 +1,347 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ wandb_version: 1
2
+
3
+ __cached__setup_devices:
4
+ desc: null
5
+ value: cpu
6
+ _n_gpu:
7
+ desc: null
8
+ value: 0
9
+ _wandb:
10
+ desc: null
11
+ value:
12
+ cli_version: 0.12.6
13
+ framework: huggingface
14
+ huggingface_version: 4.13.0.dev0
15
+ is_jupyter_run: false
16
+ is_kaggle_kernel: true
17
+ python_version: 3.8.10
18
+ start_time: 1636357016
19
+ t:
20
+ 1:
21
+ - 1
22
+ - 2
23
+ - 3
24
+ - 11
25
+ - 12
26
+ 2:
27
+ - 1
28
+ - 2
29
+ - 3
30
+ - 11
31
+ - 12
32
+ 3:
33
+ - 13
34
+ 4: 3.8.10
35
+ 5: 0.12.6
36
+ 6: 4.13.0.dev0
37
+ 8:
38
+ - 2
39
+ - 5
40
+ adafactor:
41
+ desc: null
42
+ value: false
43
+ adam_beta1:
44
+ desc: null
45
+ value: 0.9
46
+ adam_beta2:
47
+ desc: null
48
+ value: 0.999
49
+ adam_epsilon:
50
+ desc: null
51
+ value: 1.0e-08
52
+ cache_dir:
53
+ desc: null
54
+ value: null
55
+ config_name:
56
+ desc: null
57
+ value: null
58
+ dataloader_drop_last:
59
+ desc: null
60
+ value: false
61
+ dataloader_num_workers:
62
+ desc: null
63
+ value: 0
64
+ dataloader_pin_memory:
65
+ desc: null
66
+ value: true
67
+ dataset_config_name:
68
+ desc: null
69
+ value: null
70
+ dataset_name:
71
+ desc: null
72
+ value: null
73
+ ddp_find_unused_parameters:
74
+ desc: null
75
+ value: null
76
+ debug:
77
+ desc: null
78
+ value: []
79
+ deepspeed:
80
+ desc: null
81
+ value: null
82
+ disable_tqdm:
83
+ desc: null
84
+ value: false
85
+ do_eval:
86
+ desc: null
87
+ value: true
88
+ do_predict:
89
+ desc: null
90
+ value: false
91
+ do_train:
92
+ desc: null
93
+ value: true
94
+ dtype:
95
+ desc: null
96
+ value: bfloat16
97
+ eval_accumulation_steps:
98
+ desc: null
99
+ value: null
100
+ eval_steps:
101
+ desc: null
102
+ value: 5000
103
+ evaluation_strategy:
104
+ desc: null
105
+ value: IntervalStrategy.NO
106
+ fp16:
107
+ desc: null
108
+ value: false
109
+ fp16_backend:
110
+ desc: null
111
+ value: auto
112
+ fp16_full_eval:
113
+ desc: null
114
+ value: false
115
+ fp16_opt_level:
116
+ desc: null
117
+ value: O1
118
+ gradient_accumulation_steps:
119
+ desc: null
120
+ value: 1
121
+ gradient_checkpointing:
122
+ desc: null
123
+ value: false
124
+ greater_is_better:
125
+ desc: null
126
+ value: null
127
+ group_by_length:
128
+ desc: null
129
+ value: false
130
+ hub_model_id:
131
+ desc: null
132
+ value: null
133
+ hub_strategy:
134
+ desc: null
135
+ value: HubStrategy.EVERY_SAVE
136
+ hub_token:
137
+ desc: null
138
+ value: null
139
+ ignore_data_skip:
140
+ desc: null
141
+ value: false
142
+ label_names:
143
+ desc: null
144
+ value: null
145
+ label_smoothing_factor:
146
+ desc: null
147
+ value: 0.0
148
+ learning_rate:
149
+ desc: null
150
+ value: 5.0e-05
151
+ length_column_name:
152
+ desc: null
153
+ value: length
154
+ line_by_line:
155
+ desc: null
156
+ value: false
157
+ load_best_model_at_end:
158
+ desc: null
159
+ value: false
160
+ local_rank:
161
+ desc: null
162
+ value: -1
163
+ log_level:
164
+ desc: null
165
+ value: -1
166
+ log_level_replica:
167
+ desc: null
168
+ value: -1
169
+ log_on_each_node:
170
+ desc: null
171
+ value: true
172
+ logging_dir:
173
+ desc: null
174
+ value: ./runs/Nov08_07-36-00_t1v-n-ba7aa1a5-w-0
175
+ logging_first_step:
176
+ desc: null
177
+ value: false
178
+ logging_nan_inf_filter:
179
+ desc: null
180
+ value: true
181
+ logging_steps:
182
+ desc: null
183
+ value: 1000
184
+ logging_strategy:
185
+ desc: null
186
+ value: IntervalStrategy.STEPS
187
+ lr_scheduler_type:
188
+ desc: null
189
+ value: SchedulerType.LINEAR
190
+ max_grad_norm:
191
+ desc: null
192
+ value: 1.0
193
+ max_seq_length:
194
+ desc: null
195
+ value: 4096
196
+ max_steps:
197
+ desc: null
198
+ value: -1
199
+ metric_for_best_model:
200
+ desc: null
201
+ value: null
202
+ mlm_probability:
203
+ desc: null
204
+ value: 0.15
205
+ model_name_or_path:
206
+ desc: null
207
+ value: muril-bigbird
208
+ model_type:
209
+ desc: null
210
+ value: null
211
+ mp_parameters:
212
+ desc: null
213
+ value: ''
214
+ no_cuda:
215
+ desc: null
216
+ value: false
217
+ num_train_epochs:
218
+ desc: null
219
+ value: 3.0
220
+ output_dir:
221
+ desc: null
222
+ value: ./
223
+ overwrite_cache:
224
+ desc: null
225
+ value: false
226
+ overwrite_output_dir:
227
+ desc: null
228
+ value: true
229
+ pad_to_max_length:
230
+ desc: null
231
+ value: true
232
+ past_index:
233
+ desc: null
234
+ value: -1
235
+ per_device_eval_batch_size:
236
+ desc: null
237
+ value: 2
238
+ per_device_train_batch_size:
239
+ desc: null
240
+ value: 1
241
+ per_gpu_eval_batch_size:
242
+ desc: null
243
+ value: null
244
+ per_gpu_train_batch_size:
245
+ desc: null
246
+ value: null
247
+ prediction_loss_only:
248
+ desc: null
249
+ value: false
250
+ preprocessing_num_workers:
251
+ desc: null
252
+ value: null
253
+ pretokenized:
254
+ desc: null
255
+ value: true
256
+ push_to_hub:
257
+ desc: null
258
+ value: true
259
+ push_to_hub_model_id:
260
+ desc: null
261
+ value: null
262
+ push_to_hub_organization:
263
+ desc: null
264
+ value: null
265
+ push_to_hub_token:
266
+ desc: null
267
+ value: null
268
+ remove_unused_columns:
269
+ desc: null
270
+ value: true
271
+ report_to:
272
+ desc: null
273
+ value:
274
+ - tensorboard
275
+ - wandb
276
+ resume_from_checkpoint:
277
+ desc: null
278
+ value: checkpoint_100000
279
+ run_name:
280
+ desc: null
281
+ value: en-ta-hi-te-from-100k
282
+ save_on_each_node:
283
+ desc: null
284
+ value: false
285
+ save_steps:
286
+ desc: null
287
+ value: 25000
288
+ save_strategy:
289
+ desc: null
290
+ value: IntervalStrategy.STEPS
291
+ save_total_limit:
292
+ desc: null
293
+ value: 1
294
+ seed:
295
+ desc: null
296
+ value: 18
297
+ sharded_ddp:
298
+ desc: null
299
+ value: []
300
+ skip_memory_metrics:
301
+ desc: null
302
+ value: true
303
+ streaming:
304
+ desc: null
305
+ value: false
306
+ tokenizer_name:
307
+ desc: null
308
+ value: muril-bigbird
309
+ tpu_metrics_debug:
310
+ desc: null
311
+ value: false
312
+ tpu_num_cores:
313
+ desc: null
314
+ value: null
315
+ train_file:
316
+ desc: null
317
+ value: /dev/shm/data-files
318
+ train_ref_file:
319
+ desc: null
320
+ value: null
321
+ use_fast_tokenizer:
322
+ desc: null
323
+ value: true
324
+ use_legacy_prediction_loop:
325
+ desc: null
326
+ value: false
327
+ validation_file:
328
+ desc: null
329
+ value: .
330
+ validation_ref_file:
331
+ desc: null
332
+ value: null
333
+ validation_split_percentage:
334
+ desc: null
335
+ value: 5
336
+ warmup_ratio:
337
+ desc: null
338
+ value: 0.0
339
+ warmup_steps:
340
+ desc: null
341
+ value: 0
342
+ weight_decay:
343
+ desc: null
344
+ value: 0.01
345
+ xpu_backend:
346
+ desc: null
347
+ value: null
wandb/run-20211108_073656-hyqam6zi/files/output.log ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [07:37:15] - INFO - absl - A polynomial schedule was set with a non-positive `transition_steps` value; this results in a constant schedule with value `init_value`.
2
+ [07:37:15] - INFO - absl - Restoring checkpoint from checkpoint_100000
3
+ tcmalloc: large alloc 2886115328 bytes == 0x95a0a000 @ 0x7fe75a4b6680 0x7fe75a4d7824 0x5bc8a7 0x50a83b 0x50c08b 0x5f68ef 0x57107c 0x56a0ba 0x5f6343 0x56bddd 0x56a0ba 0x68d5b7 0x67cd01 0x67cd7f 0x67ce21 0x67ef47 0x6b7242 0x6b75cd 0x7fe75a2cd0b3 0x5fb18e
4
+ Traceback (most recent call last):
5
+ File "run_muril_bb_flax_mlm.py", line 612, in <module>
6
+ state = restore_checkpoint(training_args.resume_from_checkpoint, state)
7
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/training/checkpoints.py", line 274, in restore_checkpoint
8
+ results = list(results)
9
+ File "/usr/lib/python3.8/concurrent/futures/_base.py", line 619, in result_iterator
10
+ yield fs.pop().result()
11
+ File "/usr/lib/python3.8/concurrent/futures/_base.py", line 439, in result
12
+ self._condition.wait(timeout)
13
+ File "/usr/lib/python3.8/threading.py", line 302, in wait
14
+ waiter.acquire()
15
+ KeyboardInterrupt
wandb/run-20211108_073656-hyqam6zi/files/requirements.txt ADDED
@@ -0,0 +1,101 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ absl-py==0.15.0
2
+ aiohttp==3.8.0
3
+ aiosignal==1.2.0
4
+ astunparse==1.6.3
5
+ async-timeout==4.0.0
6
+ attrs==21.2.0
7
+ cachetools==4.2.4
8
+ certifi==2021.10.8
9
+ charset-normalizer==2.0.7
10
+ chex==0.0.8
11
+ click==8.0.3
12
+ configparser==5.1.0
13
+ cycler==0.11.0
14
+ datasets==1.15.1
15
+ dill==0.3.4
16
+ dm-tree==0.1.6
17
+ docker-pycreds==0.4.0
18
+ filelock==3.3.2
19
+ flatbuffers==2.0
20
+ flax==0.3.6
21
+ frozenlist==1.2.0
22
+ fsspec==2021.11.0
23
+ gast==0.4.0
24
+ gitdb==4.0.9
25
+ gitpython==3.1.24
26
+ google-auth-oauthlib==0.4.6
27
+ google-auth==2.3.3
28
+ google-pasta==0.2.0
29
+ grpcio==1.41.1
30
+ h5py==3.5.0
31
+ huggingface-hub==0.1.1
32
+ idna==3.3
33
+ jax==0.2.24
34
+ jaxlib==0.1.73
35
+ joblib==1.1.0
36
+ kaggle==1.5.12
37
+ keras-preprocessing==1.1.2
38
+ keras==2.7.0
39
+ kiwisolver==1.3.2
40
+ libclang==12.0.0
41
+ libtpu-nightly==0.1.dev20211018
42
+ markdown==3.3.4
43
+ matplotlib==3.4.3
44
+ msgpack==1.0.2
45
+ multidict==5.2.0
46
+ multiprocess==0.70.12.2
47
+ numpy==1.21.4
48
+ oauthlib==3.1.1
49
+ opt-einsum==3.3.0
50
+ optax==0.0.9
51
+ packaging==21.2
52
+ pandas==1.3.4
53
+ pathtools==0.1.2
54
+ pillow==8.4.0
55
+ pip==21.3.1
56
+ pkg-resources==0.0.0
57
+ promise==2.3
58
+ protobuf==3.19.1
59
+ psutil==5.8.0
60
+ pyarrow==6.0.0
61
+ pyasn1-modules==0.2.8
62
+ pyasn1==0.4.8
63
+ pyparsing==2.4.7
64
+ python-dateutil==2.8.2
65
+ python-slugify==5.0.2
66
+ pytz==2021.3
67
+ pyyaml==6.0
68
+ regex==2021.11.2
69
+ requests-oauthlib==1.3.0
70
+ requests==2.26.0
71
+ rsa==4.7.2
72
+ sacremoses==0.0.46
73
+ scipy==1.7.1
74
+ sentry-sdk==1.4.3
75
+ setuptools==44.0.0
76
+ shortuuid==1.0.1
77
+ six==1.16.0
78
+ smmap==5.0.0
79
+ subprocess32==3.5.4
80
+ tensorboard-data-server==0.6.1
81
+ tensorboard-plugin-wit==1.8.0
82
+ tensorboard==2.7.0
83
+ tensorflow-estimator==2.7.0
84
+ tensorflow-io-gcs-filesystem==0.21.0
85
+ tensorflow==2.7.0
86
+ termcolor==1.1.0
87
+ text-unidecode==1.3
88
+ tokenizers==0.10.3
89
+ toolz==0.11.1
90
+ torch==1.10.0
91
+ tqdm==4.62.3
92
+ transformers==4.13.0.dev0
93
+ typing-extensions==3.10.0.2
94
+ urllib3==1.26.7
95
+ wandb==0.12.6
96
+ werkzeug==2.0.2
97
+ wheel==0.37.0
98
+ wrapt==1.13.3
99
+ xxhash==2.0.2
100
+ yarl==1.7.2
101
+ yaspin==2.1.0
wandb/run-20211108_073656-hyqam6zi/files/wandb-metadata.json ADDED
@@ -0,0 +1,69 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29",
3
+ "python": "3.8.10",
4
+ "heartbeatAt": "2021-11-08T07:36:59.308384",
5
+ "startedAt": "2021-11-08T07:36:56.467866",
6
+ "docker": null,
7
+ "cpu_count": 96,
8
+ "cuda": null,
9
+ "args": [
10
+ "--model_name_or_path",
11
+ "muril-bigbird",
12
+ "--tokenizer_name",
13
+ "muril-bigbird",
14
+ "--dtype",
15
+ "bfloat16",
16
+ "--pretokenized",
17
+ "--resume_from_checkpoint",
18
+ "checkpoint_100000",
19
+ "--train_file",
20
+ "/dev/shm/data-files",
21
+ "--validation_file",
22
+ ".",
23
+ "--max_seq_length",
24
+ "4096",
25
+ "--pad_to_max_length",
26
+ "--output_dir",
27
+ "./",
28
+ "--overwrite_output_dir",
29
+ "--do_train",
30
+ "--do_eval",
31
+ "--eval_steps",
32
+ "5000",
33
+ "--per_device_train_batch_size",
34
+ "1",
35
+ "--per_device_eval_batch_size",
36
+ "2",
37
+ "--learning_rate",
38
+ "5e-5",
39
+ "--weight_decay",
40
+ "0.01",
41
+ "--num_train_epochs",
42
+ "3",
43
+ "--warmup_steps",
44
+ "0",
45
+ "--logging_steps",
46
+ "1000",
47
+ "--save_steps",
48
+ "25000",
49
+ "--save_total_limit",
50
+ "1",
51
+ "--seed",
52
+ "18",
53
+ "--push_to_hub",
54
+ "--run_name",
55
+ "en-ta-hi-te-from-100k"
56
+ ],
57
+ "state": "running",
58
+ "program": "run_muril_bb_flax_mlm.py",
59
+ "codePath": "run_muril_bb_flax_mlm.py",
60
+ "git": {
61
+ "remote": "https://huggingface.co/nbroad/muril-bigbird-base-1mil",
62
+ "commit": "5fca06e3f8deb06ee5f938513b5f093fc3b66a62"
63
+ },
64
+ "email": "nicholas@nmbroad.com",
65
+ "root": "/home/nicholasbroad/muril-bigbird-base-1mil",
66
+ "host": "t1v-n-ba7aa1a5-w-0",
67
+ "username": "nicholasbroad",
68
+ "executable": "/home/nicholasbroad/venv/bin/python"
69
+ }
wandb/run-20211108_073656-hyqam6zi/files/wandb-summary.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"_wandb": {"runtime": 25}}
wandb/run-20211108_073656-hyqam6zi/logs/debug-internal.log ADDED
@@ -0,0 +1,145 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2021-11-08 07:36:57,125 INFO MainThread:30896 [internal.py:wandb_internal():87] W&B internal server running at pid: 30896, started at: 2021-11-08 07:36:57.125057
2
+ 2021-11-08 07:36:57,127 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: check_version
3
+ 2021-11-08 07:36:57,127 INFO WriterThread:30896 [datastore.py:open_for_write():77] open: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/run-hyqam6zi.wandb
4
+ 2021-11-08 07:36:57,128 DEBUG SenderThread:30896 [sender.py:send():238] send: header
5
+ 2021-11-08 07:36:57,128 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: check_version
6
+ 2021-11-08 07:36:57,165 DEBUG SenderThread:30896 [sender.py:send():238] send: run
7
+ 2021-11-08 07:36:57,327 INFO SenderThread:30896 [dir_watcher.py:__init__():169] watching files in: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/files
8
+ 2021-11-08 07:36:57,327 INFO SenderThread:30896 [sender.py:_start_run_threads():825] run started: hyqam6zi with start time 1636357016
9
+ 2021-11-08 07:36:57,327 DEBUG SenderThread:30896 [sender.py:send():238] send: summary
10
+ 2021-11-08 07:36:57,327 INFO SenderThread:30896 [sender.py:_save_file():960] saving file wandb-summary.json with policy end
11
+ 2021-11-08 07:36:57,328 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: run_start
12
+ 2021-11-08 07:36:58,329 INFO Thread-8 :30896 [dir_watcher.py:_on_file_created():217] file/dir created: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/files/wandb-summary.json
13
+ 2021-11-08 07:36:59,308 DEBUG HandlerThread:30896 [meta.py:__init__():40] meta init
14
+ 2021-11-08 07:36:59,308 DEBUG HandlerThread:30896 [meta.py:__init__():54] meta init done
15
+ 2021-11-08 07:36:59,308 DEBUG HandlerThread:30896 [meta.py:probe():214] probe
16
+ 2021-11-08 07:36:59,309 DEBUG HandlerThread:30896 [meta.py:_setup_git():204] setup git
17
+ 2021-11-08 07:36:59,338 DEBUG HandlerThread:30896 [meta.py:_setup_git():211] setup git done
18
+ 2021-11-08 07:36:59,338 DEBUG HandlerThread:30896 [meta.py:_save_pip():58] save pip
19
+ 2021-11-08 07:36:59,339 DEBUG HandlerThread:30896 [meta.py:_save_pip():72] save pip done
20
+ 2021-11-08 07:36:59,339 DEBUG HandlerThread:30896 [meta.py:probe():252] probe done
21
+ 2021-11-08 07:36:59,342 DEBUG SenderThread:30896 [sender.py:send():238] send: files
22
+ 2021-11-08 07:36:59,342 INFO SenderThread:30896 [sender.py:_save_file():960] saving file wandb-metadata.json with policy now
23
+ 2021-11-08 07:36:59,347 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: stop_status
24
+ 2021-11-08 07:36:59,347 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: stop_status
25
+ 2021-11-08 07:36:59,480 DEBUG SenderThread:30896 [sender.py:send():238] send: config
26
+ 2021-11-08 07:36:59,480 DEBUG SenderThread:30896 [sender.py:send():238] send: config
27
+ 2021-11-08 07:36:59,480 DEBUG SenderThread:30896 [sender.py:send():238] send: config
28
+ 2021-11-08 07:36:59,806 INFO Thread-11 :30896 [upload_job.py:push():137] Uploaded file /tmp/tmptc9slxn9wandb/2u6gi7c9-wandb-metadata.json
29
+ 2021-11-08 07:37:00,336 INFO Thread-8 :30896 [dir_watcher.py:_on_file_created():217] file/dir created: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/files/requirements.txt
30
+ 2021-11-08 07:37:00,337 INFO Thread-8 :30896 [dir_watcher.py:_on_file_created():217] file/dir created: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/files/wandb-metadata.json
31
+ 2021-11-08 07:37:00,337 INFO Thread-8 :30896 [dir_watcher.py:_on_file_created():217] file/dir created: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/files/output.log
32
+ 2021-11-08 07:37:14,958 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: stop_status
33
+ 2021-11-08 07:37:14,958 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: stop_status
34
+ 2021-11-08 07:37:19,976 WARNING MainThread:30896 [internal.py:wandb_internal():146] Internal process interrupt: 1
35
+ 2021-11-08 07:37:20,344 INFO Thread-8 :30896 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/files/output.log
36
+ 2021-11-08 07:37:22,344 INFO Thread-8 :30896 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/files/output.log
37
+ 2021-11-08 07:37:22,876 DEBUG SenderThread:30896 [sender.py:send():238] send: telemetry
38
+ 2021-11-08 07:37:22,876 DEBUG SenderThread:30896 [sender.py:send():238] send: exit
39
+ 2021-11-08 07:37:22,877 INFO SenderThread:30896 [sender.py:send_exit():370] handling exit code: 255
40
+ 2021-11-08 07:37:22,877 INFO SenderThread:30896 [sender.py:send_exit():372] handling runtime: 25
41
+ 2021-11-08 07:37:22,877 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: poll_exit
42
+ 2021-11-08 07:37:22,878 INFO SenderThread:30896 [sender.py:_save_file():960] saving file wandb-summary.json with policy end
43
+ 2021-11-08 07:37:22,878 INFO SenderThread:30896 [sender.py:send_exit():382] send defer
44
+ 2021-11-08 07:37:22,878 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: poll_exit
45
+ 2021-11-08 07:37:22,879 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: defer
46
+ 2021-11-08 07:37:22,879 INFO HandlerThread:30896 [handler.py:handle_request_defer():147] handle defer: 0
47
+ 2021-11-08 07:37:22,879 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: defer
48
+ 2021-11-08 07:37:22,879 INFO SenderThread:30896 [sender.py:send_request_defer():391] handle sender defer: 0
49
+ 2021-11-08 07:37:22,879 INFO SenderThread:30896 [sender.py:transition_state():395] send defer: 1
50
+ 2021-11-08 07:37:22,879 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: defer
51
+ 2021-11-08 07:37:22,879 INFO HandlerThread:30896 [handler.py:handle_request_defer():147] handle defer: 1
52
+ 2021-11-08 07:37:22,976 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: defer
53
+ 2021-11-08 07:37:22,976 INFO SenderThread:30896 [sender.py:send_request_defer():391] handle sender defer: 1
54
+ 2021-11-08 07:37:22,976 INFO SenderThread:30896 [sender.py:transition_state():395] send defer: 2
55
+ 2021-11-08 07:37:22,976 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: defer
56
+ 2021-11-08 07:37:22,976 INFO HandlerThread:30896 [handler.py:handle_request_defer():147] handle defer: 2
57
+ 2021-11-08 07:37:22,976 DEBUG SenderThread:30896 [sender.py:send():238] send: stats
58
+ 2021-11-08 07:37:22,976 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: defer
59
+ 2021-11-08 07:37:22,977 INFO SenderThread:30896 [sender.py:send_request_defer():391] handle sender defer: 2
60
+ 2021-11-08 07:37:22,977 INFO SenderThread:30896 [sender.py:transition_state():395] send defer: 3
61
+ 2021-11-08 07:37:22,977 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: defer
62
+ 2021-11-08 07:37:22,977 INFO HandlerThread:30896 [handler.py:handle_request_defer():147] handle defer: 3
63
+ 2021-11-08 07:37:22,977 DEBUG SenderThread:30896 [sender.py:send():238] send: summary
64
+ 2021-11-08 07:37:22,978 INFO SenderThread:30896 [sender.py:_save_file():960] saving file wandb-summary.json with policy end
65
+ 2021-11-08 07:37:22,978 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: defer
66
+ 2021-11-08 07:37:22,978 INFO SenderThread:30896 [sender.py:send_request_defer():391] handle sender defer: 3
67
+ 2021-11-08 07:37:22,978 INFO SenderThread:30896 [sender.py:transition_state():395] send defer: 4
68
+ 2021-11-08 07:37:22,978 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: defer
69
+ 2021-11-08 07:37:22,978 INFO HandlerThread:30896 [handler.py:handle_request_defer():147] handle defer: 4
70
+ 2021-11-08 07:37:22,979 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: defer
71
+ 2021-11-08 07:37:22,979 INFO SenderThread:30896 [sender.py:send_request_defer():391] handle sender defer: 4
72
+ 2021-11-08 07:37:22,983 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: poll_exit
73
+ 2021-11-08 07:37:23,162 INFO SenderThread:30896 [sender.py:transition_state():395] send defer: 5
74
+ 2021-11-08 07:37:23,162 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: poll_exit
75
+ 2021-11-08 07:37:23,163 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: defer
76
+ 2021-11-08 07:37:23,163 INFO HandlerThread:30896 [handler.py:handle_request_defer():147] handle defer: 5
77
+ 2021-11-08 07:37:23,163 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: defer
78
+ 2021-11-08 07:37:23,163 INFO SenderThread:30896 [sender.py:send_request_defer():391] handle sender defer: 5
79
+ 2021-11-08 07:37:23,163 INFO SenderThread:30896 [dir_watcher.py:finish():283] shutting down directory watcher
80
+ 2021-11-08 07:37:23,264 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: poll_exit
81
+ 2021-11-08 07:37:23,345 INFO Thread-8 :30896 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/files/wandb-summary.json
82
+ 2021-11-08 07:37:23,345 INFO SenderThread:30896 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/files/config.yaml
83
+ 2021-11-08 07:37:23,346 INFO SenderThread:30896 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/files/output.log
84
+ 2021-11-08 07:37:23,346 INFO SenderThread:30896 [dir_watcher.py:finish():313] scan: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/files
85
+ 2021-11-08 07:37:23,346 INFO SenderThread:30896 [dir_watcher.py:finish():327] scan save: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/files/requirements.txt requirements.txt
86
+ 2021-11-08 07:37:23,346 INFO SenderThread:30896 [dir_watcher.py:finish():327] scan save: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/files/output.log output.log
87
+ 2021-11-08 07:37:23,346 INFO SenderThread:30896 [dir_watcher.py:finish():327] scan save: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/files/wandb-metadata.json wandb-metadata.json
88
+ 2021-11-08 07:37:23,346 INFO SenderThread:30896 [dir_watcher.py:finish():327] scan save: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/files/config.yaml config.yaml
89
+ 2021-11-08 07:37:23,347 INFO SenderThread:30896 [dir_watcher.py:finish():327] scan save: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/files/wandb-summary.json wandb-summary.json
90
+ 2021-11-08 07:37:23,352 INFO SenderThread:30896 [sender.py:transition_state():395] send defer: 6
91
+ 2021-11-08 07:37:23,353 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: poll_exit
92
+ 2021-11-08 07:37:23,356 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: defer
93
+ 2021-11-08 07:37:23,356 INFO HandlerThread:30896 [handler.py:handle_request_defer():147] handle defer: 6
94
+ 2021-11-08 07:37:23,359 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: defer
95
+ 2021-11-08 07:37:23,359 INFO SenderThread:30896 [sender.py:send_request_defer():391] handle sender defer: 6
96
+ 2021-11-08 07:37:23,359 INFO SenderThread:30896 [file_pusher.py:finish():177] shutting down file pusher
97
+ 2021-11-08 07:37:23,454 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: poll_exit
98
+ 2021-11-08 07:37:23,454 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: poll_exit
99
+ 2021-11-08 07:37:23,556 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: poll_exit
100
+ 2021-11-08 07:37:23,556 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: poll_exit
101
+ 2021-11-08 07:37:23,658 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: poll_exit
102
+ 2021-11-08 07:37:23,658 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: poll_exit
103
+ 2021-11-08 07:37:23,759 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: poll_exit
104
+ 2021-11-08 07:37:23,760 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: poll_exit
105
+ 2021-11-08 07:37:23,853 INFO Thread-14 :30896 [upload_job.py:push():137] Uploaded file /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/files/config.yaml
106
+ 2021-11-08 07:37:23,861 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: poll_exit
107
+ 2021-11-08 07:37:23,861 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: poll_exit
108
+ 2021-11-08 07:37:23,867 INFO Thread-13 :30896 [upload_job.py:push():137] Uploaded file /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/files/output.log
109
+ 2021-11-08 07:37:23,908 INFO Thread-12 :30896 [upload_job.py:push():137] Uploaded file /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/files/requirements.txt
110
+ 2021-11-08 07:37:23,921 INFO Thread-15 :30896 [upload_job.py:push():137] Uploaded file /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/files/wandb-summary.json
111
+ 2021-11-08 07:37:23,963 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: poll_exit
112
+ 2021-11-08 07:37:23,963 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: poll_exit
113
+ 2021-11-08 07:37:24,064 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: poll_exit
114
+ 2021-11-08 07:37:24,065 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: poll_exit
115
+ 2021-11-08 07:37:24,122 INFO Thread-7 :30896 [sender.py:transition_state():395] send defer: 7
116
+ 2021-11-08 07:37:24,122 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: defer
117
+ 2021-11-08 07:37:24,122 INFO HandlerThread:30896 [handler.py:handle_request_defer():147] handle defer: 7
118
+ 2021-11-08 07:37:24,122 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: defer
119
+ 2021-11-08 07:37:24,122 INFO SenderThread:30896 [sender.py:send_request_defer():391] handle sender defer: 7
120
+ 2021-11-08 07:37:24,166 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: poll_exit
121
+ 2021-11-08 07:37:24,407 INFO SenderThread:30896 [sender.py:transition_state():395] send defer: 8
122
+ 2021-11-08 07:37:24,408 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: poll_exit
123
+ 2021-11-08 07:37:24,408 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: defer
124
+ 2021-11-08 07:37:24,408 INFO HandlerThread:30896 [handler.py:handle_request_defer():147] handle defer: 8
125
+ 2021-11-08 07:37:24,408 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: defer
126
+ 2021-11-08 07:37:24,409 INFO SenderThread:30896 [sender.py:send_request_defer():391] handle sender defer: 8
127
+ 2021-11-08 07:37:24,409 INFO SenderThread:30896 [sender.py:transition_state():395] send defer: 9
128
+ 2021-11-08 07:37:24,409 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: defer
129
+ 2021-11-08 07:37:24,409 INFO HandlerThread:30896 [handler.py:handle_request_defer():147] handle defer: 9
130
+ 2021-11-08 07:37:24,409 DEBUG SenderThread:30896 [sender.py:send():238] send: final
131
+ 2021-11-08 07:37:24,410 DEBUG SenderThread:30896 [sender.py:send():238] send: footer
132
+ 2021-11-08 07:37:24,410 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: defer
133
+ 2021-11-08 07:37:24,410 INFO SenderThread:30896 [sender.py:send_request_defer():391] handle sender defer: 9
134
+ 2021-11-08 07:37:24,509 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: poll_exit
135
+ 2021-11-08 07:37:24,509 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: poll_exit
136
+ 2021-11-08 07:37:24,509 INFO SenderThread:30896 [file_pusher.py:join():182] waiting for file pusher
137
+ 2021-11-08 07:37:24,762 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: get_summary
138
+ 2021-11-08 07:37:24,762 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: sampled_history
139
+ 2021-11-08 07:37:24,763 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: shutdown
140
+ 2021-11-08 07:37:24,763 INFO HandlerThread:30896 [handler.py:finish():731] shutting down handler
141
+ 2021-11-08 07:37:25,410 INFO WriterThread:30896 [datastore.py:close():281] close: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/run-hyqam6zi.wandb
142
+ 2021-11-08 07:37:25,761 INFO SenderThread:30896 [sender.py:finish():1091] shutting down sender
143
+ 2021-11-08 07:37:25,761 INFO SenderThread:30896 [file_pusher.py:finish():177] shutting down file pusher
144
+ 2021-11-08 07:37:25,761 INFO SenderThread:30896 [file_pusher.py:join():182] waiting for file pusher
145
+ 2021-11-08 07:37:25,763 INFO MainThread:30896 [internal.py:handle_exit():77] Internal process exited
wandb/run-20211108_073656-hyqam6zi/logs/debug.log ADDED
@@ -0,0 +1,129 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2021-11-08 07:36:56,476 INFO MainThread:29588 [wandb_setup.py:_flush():71] setting env: {}
2
+ 2021-11-08 07:36:56,477 INFO MainThread:29588 [wandb_setup.py:_flush():71] setting login settings: {}
3
+ 2021-11-08 07:36:56,477 INFO MainThread:29588 [wandb_init.py:_log_setup():357] Logging user logs to /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/logs/debug.log
4
+ 2021-11-08 07:36:56,477 INFO MainThread:29588 [wandb_init.py:_log_setup():358] Logging internal logs to /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/logs/debug-internal.log
5
+ 2021-11-08 07:36:56,477 INFO MainThread:29588 [wandb_init.py:init():390] calling init triggers
6
+ 2021-11-08 07:36:56,477 INFO MainThread:29588 [wandb_init.py:init():395] wandb.init called with sweep_config: {}
7
+ config: {}
8
+ 2021-11-08 07:36:56,477 INFO MainThread:29588 [wandb_init.py:init():435] starting backend
9
+ 2021-11-08 07:36:56,477 INFO MainThread:29588 [backend.py:_multiprocessing_setup():95] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
10
+ 2021-11-08 07:36:56,515 INFO MainThread:29588 [backend.py:ensure_launched():199] starting backend process...
11
+ 2021-11-08 07:36:56,551 INFO MainThread:29588 [backend.py:ensure_launched():204] started backend process with pid: 30896
12
+ 2021-11-08 07:36:56,553 INFO MainThread:29588 [wandb_init.py:init():444] backend started and connected
13
+ 2021-11-08 07:36:56,555 INFO MainThread:29588 [wandb_init.py:init():503] updated telemetry
14
+ 2021-11-08 07:36:56,638 INFO MainThread:29588 [wandb_init.py:init():533] communicating current version
15
+ 2021-11-08 07:36:57,164 INFO MainThread:29588 [wandb_init.py:init():538] got version response
16
+ 2021-11-08 07:36:57,164 INFO MainThread:29588 [wandb_init.py:init():548] communicating run to backend with 30 second timeout
17
+ 2021-11-08 07:36:57,328 INFO MainThread:29588 [wandb_init.py:init():576] starting run threads in backend
18
+ 2021-11-08 07:36:59,347 INFO MainThread:29588 [wandb_run.py:_console_start():1693] atexit reg
19
+ 2021-11-08 07:36:59,348 INFO MainThread:29588 [wandb_run.py:_redirect():1567] redirect: SettingsConsole.REDIRECT
20
+ 2021-11-08 07:36:59,348 INFO MainThread:29588 [wandb_run.py:_redirect():1572] Redirecting console.
21
+ 2021-11-08 07:36:59,350 INFO MainThread:29588 [wandb_run.py:_redirect():1628] Redirects installed.
22
+ 2021-11-08 07:36:59,350 INFO MainThread:29588 [wandb_init.py:init():603] run started, returning control to user process
23
+ 2021-11-08 07:36:59,355 INFO MainThread:29588 [wandb_run.py:_config_callback():911] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': True, 'do_eval': True, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 2, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 5e-05, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 3.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 0, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Nov08_07-36-00_t1v-n-ba7aa1a5-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 1000, 'logging_nan_inf_filter': True, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 25000, 'save_total_limit': 1, 'save_on_each_node': False, 'no_cuda': False, 'seed': 18, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'xpu_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 5000, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': 'en-ta-hi-te-from-100k', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': 'checkpoint_100000', 'hub_model_id': None, 'hub_strategy': 'HubStrategy.EVERY_SAVE', 'hub_token': None, 'gradient_checkpointing': False, 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': '', '_n_gpu': 0, '__cached__setup_devices': 'cpu'}
24
+ 2021-11-08 07:36:59,357 INFO MainThread:29588 [wandb_run.py:_config_callback():911] config_cb None None {'model_name_or_path': 'muril-bigbird', 'model_type': None, 'config_name': None, 'tokenizer_name': 'muril-bigbird', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'bfloat16'}
25
+ 2021-11-08 07:36:59,358 INFO MainThread:29588 [wandb_run.py:_config_callback():911] config_cb None None {'dataset_name': None, 'pretokenized': True, 'streaming': False, 'dataset_config_name': None, 'train_file': '/dev/shm/data-files', 'validation_file': '.', 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': None, 'mlm_probability': 0.15, 'pad_to_max_length': True, 'line_by_line': False}
26
+ 2021-11-08 07:37:20,244 INFO MainThread:29588 [wandb_run.py:_atexit_cleanup():1663] got exitcode: 255
27
+ 2021-11-08 07:37:20,245 INFO MainThread:29588 [wandb_run.py:_restore():1635] restore
28
+ 2021-11-08 07:37:22,879 INFO MainThread:29588 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
29
+ wandb_count: 1
30
+ }
31
+ pusher_stats {
32
+ uploaded_bytes: 1825
33
+ total_bytes: 1825
34
+ }
35
+
36
+ 2021-11-08 07:37:23,163 INFO MainThread:29588 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
37
+ wandb_count: 1
38
+ }
39
+ pusher_stats {
40
+ uploaded_bytes: 1825
41
+ total_bytes: 1825
42
+ }
43
+
44
+ 2021-11-08 07:37:23,353 INFO MainThread:29588 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
45
+ wandb_count: 4
46
+ }
47
+ pusher_stats {
48
+ uploaded_bytes: 1825
49
+ total_bytes: 9944
50
+ }
51
+
52
+ 2021-11-08 07:37:23,455 INFO MainThread:29588 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
53
+ wandb_count: 5
54
+ }
55
+ pusher_stats {
56
+ uploaded_bytes: 1825
57
+ total_bytes: 9971
58
+ }
59
+
60
+ 2021-11-08 07:37:23,557 INFO MainThread:29588 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
61
+ wandb_count: 5
62
+ }
63
+ pusher_stats {
64
+ uploaded_bytes: 9971
65
+ total_bytes: 9971
66
+ }
67
+
68
+ 2021-11-08 07:37:23,659 INFO MainThread:29588 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
69
+ wandb_count: 5
70
+ }
71
+ pusher_stats {
72
+ uploaded_bytes: 9971
73
+ total_bytes: 9971
74
+ }
75
+
76
+ 2021-11-08 07:37:23,760 INFO MainThread:29588 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
77
+ wandb_count: 5
78
+ }
79
+ pusher_stats {
80
+ uploaded_bytes: 9971
81
+ total_bytes: 9971
82
+ }
83
+
84
+ 2021-11-08 07:37:23,862 INFO MainThread:29588 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
85
+ wandb_count: 5
86
+ }
87
+ pusher_stats {
88
+ uploaded_bytes: 9971
89
+ total_bytes: 9971
90
+ }
91
+
92
+ 2021-11-08 07:37:23,963 INFO MainThread:29588 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
93
+ wandb_count: 5
94
+ }
95
+ pusher_stats {
96
+ uploaded_bytes: 9971
97
+ total_bytes: 9971
98
+ }
99
+
100
+ 2021-11-08 07:37:24,065 INFO MainThread:29588 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
101
+ wandb_count: 5
102
+ }
103
+ pusher_stats {
104
+ uploaded_bytes: 9971
105
+ total_bytes: 9971
106
+ }
107
+
108
+ 2021-11-08 07:37:24,408 INFO MainThread:29588 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
109
+ wandb_count: 5
110
+ }
111
+ pusher_stats {
112
+ uploaded_bytes: 9971
113
+ total_bytes: 9971
114
+ }
115
+
116
+ 2021-11-08 07:37:24,761 INFO MainThread:29588 [wandb_run.py:_wait_for_finish():1793] got exit ret: done: true
117
+ exit_result {
118
+ }
119
+ file_counts {
120
+ wandb_count: 5
121
+ }
122
+ pusher_stats {
123
+ uploaded_bytes: 9971
124
+ total_bytes: 9971
125
+ }
126
+ local_info {
127
+ }
128
+
129
+ 2021-11-08 07:37:26,128 INFO MainThread:29588 [wandb_run.py:_append_files():2061] logging synced files
wandb/run-20211108_073656-hyqam6zi/run-hyqam6zi.wandb ADDED
Binary file (4.85 kB). View file
 
wandb/run-20211108_080537-1gq01aek/files/config.yaml ADDED
@@ -0,0 +1,347 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ wandb_version: 1
2
+
3
+ __cached__setup_devices:
4
+ desc: null
5
+ value: cpu
6
+ _n_gpu:
7
+ desc: null
8
+ value: 0
9
+ _wandb:
10
+ desc: null
11
+ value:
12
+ cli_version: 0.12.6
13
+ framework: huggingface
14
+ huggingface_version: 4.13.0.dev0
15
+ is_jupyter_run: false
16
+ is_kaggle_kernel: true
17
+ python_version: 3.8.10
18
+ start_time: 1636358737
19
+ t:
20
+ 1:
21
+ - 1
22
+ - 2
23
+ - 3
24
+ - 11
25
+ - 12
26
+ 2:
27
+ - 1
28
+ - 2
29
+ - 3
30
+ - 11
31
+ - 12
32
+ 3:
33
+ - 13
34
+ 4: 3.8.10
35
+ 5: 0.12.6
36
+ 6: 4.13.0.dev0
37
+ 8:
38
+ - 2
39
+ - 5
40
+ adafactor:
41
+ desc: null
42
+ value: false
43
+ adam_beta1:
44
+ desc: null
45
+ value: 0.9
46
+ adam_beta2:
47
+ desc: null
48
+ value: 0.999
49
+ adam_epsilon:
50
+ desc: null
51
+ value: 1.0e-08
52
+ cache_dir:
53
+ desc: null
54
+ value: null
55
+ config_name:
56
+ desc: null
57
+ value: null
58
+ dataloader_drop_last:
59
+ desc: null
60
+ value: false
61
+ dataloader_num_workers:
62
+ desc: null
63
+ value: 0
64
+ dataloader_pin_memory:
65
+ desc: null
66
+ value: true
67
+ dataset_config_name:
68
+ desc: null
69
+ value: null
70
+ dataset_name:
71
+ desc: null
72
+ value: null
73
+ ddp_find_unused_parameters:
74
+ desc: null
75
+ value: null
76
+ debug:
77
+ desc: null
78
+ value: []
79
+ deepspeed:
80
+ desc: null
81
+ value: null
82
+ disable_tqdm:
83
+ desc: null
84
+ value: false
85
+ do_eval:
86
+ desc: null
87
+ value: true
88
+ do_predict:
89
+ desc: null
90
+ value: false
91
+ do_train:
92
+ desc: null
93
+ value: true
94
+ dtype:
95
+ desc: null
96
+ value: bfloat16
97
+ eval_accumulation_steps:
98
+ desc: null
99
+ value: null
100
+ eval_steps:
101
+ desc: null
102
+ value: 5000
103
+ evaluation_strategy:
104
+ desc: null
105
+ value: IntervalStrategy.NO
106
+ fp16:
107
+ desc: null
108
+ value: false
109
+ fp16_backend:
110
+ desc: null
111
+ value: auto
112
+ fp16_full_eval:
113
+ desc: null
114
+ value: false
115
+ fp16_opt_level:
116
+ desc: null
117
+ value: O1
118
+ gradient_accumulation_steps:
119
+ desc: null
120
+ value: 1
121
+ gradient_checkpointing:
122
+ desc: null
123
+ value: false
124
+ greater_is_better:
125
+ desc: null
126
+ value: null
127
+ group_by_length:
128
+ desc: null
129
+ value: false
130
+ hub_model_id:
131
+ desc: null
132
+ value: null
133
+ hub_strategy:
134
+ desc: null
135
+ value: HubStrategy.EVERY_SAVE
136
+ hub_token:
137
+ desc: null
138
+ value: null
139
+ ignore_data_skip:
140
+ desc: null
141
+ value: false
142
+ label_names:
143
+ desc: null
144
+ value: null
145
+ label_smoothing_factor:
146
+ desc: null
147
+ value: 0.0
148
+ learning_rate:
149
+ desc: null
150
+ value: 5.0e-05
151
+ length_column_name:
152
+ desc: null
153
+ value: length
154
+ line_by_line:
155
+ desc: null
156
+ value: false
157
+ load_best_model_at_end:
158
+ desc: null
159
+ value: false
160
+ local_rank:
161
+ desc: null
162
+ value: -1
163
+ log_level:
164
+ desc: null
165
+ value: -1
166
+ log_level_replica:
167
+ desc: null
168
+ value: -1
169
+ log_on_each_node:
170
+ desc: null
171
+ value: true
172
+ logging_dir:
173
+ desc: null
174
+ value: ./runs/Nov08_08-04-38_t1v-n-ba7aa1a5-w-0
175
+ logging_first_step:
176
+ desc: null
177
+ value: false
178
+ logging_nan_inf_filter:
179
+ desc: null
180
+ value: true
181
+ logging_steps:
182
+ desc: null
183
+ value: 1000
184
+ logging_strategy:
185
+ desc: null
186
+ value: IntervalStrategy.STEPS
187
+ lr_scheduler_type:
188
+ desc: null
189
+ value: SchedulerType.LINEAR
190
+ max_grad_norm:
191
+ desc: null
192
+ value: 1.0
193
+ max_seq_length:
194
+ desc: null
195
+ value: 4096
196
+ max_steps:
197
+ desc: null
198
+ value: -1
199
+ metric_for_best_model:
200
+ desc: null
201
+ value: null
202
+ mlm_probability:
203
+ desc: null
204
+ value: 0.15
205
+ model_name_or_path:
206
+ desc: null
207
+ value: muril-bigbird
208
+ model_type:
209
+ desc: null
210
+ value: null
211
+ mp_parameters:
212
+ desc: null
213
+ value: ''
214
+ no_cuda:
215
+ desc: null
216
+ value: false
217
+ num_train_epochs:
218
+ desc: null
219
+ value: 3.0
220
+ output_dir:
221
+ desc: null
222
+ value: ./
223
+ overwrite_cache:
224
+ desc: null
225
+ value: false
226
+ overwrite_output_dir:
227
+ desc: null
228
+ value: true
229
+ pad_to_max_length:
230
+ desc: null
231
+ value: true
232
+ past_index:
233
+ desc: null
234
+ value: -1
235
+ per_device_eval_batch_size:
236
+ desc: null
237
+ value: 2
238
+ per_device_train_batch_size:
239
+ desc: null
240
+ value: 1
241
+ per_gpu_eval_batch_size:
242
+ desc: null
243
+ value: null
244
+ per_gpu_train_batch_size:
245
+ desc: null
246
+ value: null
247
+ prediction_loss_only:
248
+ desc: null
249
+ value: false
250
+ preprocessing_num_workers:
251
+ desc: null
252
+ value: null
253
+ pretokenized:
254
+ desc: null
255
+ value: true
256
+ push_to_hub:
257
+ desc: null
258
+ value: true
259
+ push_to_hub_model_id:
260
+ desc: null
261
+ value: null
262
+ push_to_hub_organization:
263
+ desc: null
264
+ value: null
265
+ push_to_hub_token:
266
+ desc: null
267
+ value: null
268
+ remove_unused_columns:
269
+ desc: null
270
+ value: true
271
+ report_to:
272
+ desc: null
273
+ value:
274
+ - tensorboard
275
+ - wandb
276
+ resume_from_checkpoint:
277
+ desc: null
278
+ value: checkpoint_100000
279
+ run_name:
280
+ desc: null
281
+ value: en-ta-hi-te-from-100k
282
+ save_on_each_node:
283
+ desc: null
284
+ value: false
285
+ save_steps:
286
+ desc: null
287
+ value: 25000
288
+ save_strategy:
289
+ desc: null
290
+ value: IntervalStrategy.STEPS
291
+ save_total_limit:
292
+ desc: null
293
+ value: 1
294
+ seed:
295
+ desc: null
296
+ value: 18
297
+ sharded_ddp:
298
+ desc: null
299
+ value: []
300
+ skip_memory_metrics:
301
+ desc: null
302
+ value: true
303
+ streaming:
304
+ desc: null
305
+ value: false
306
+ tokenizer_name:
307
+ desc: null
308
+ value: muril-bigbird
309
+ tpu_metrics_debug:
310
+ desc: null
311
+ value: false
312
+ tpu_num_cores:
313
+ desc: null
314
+ value: null
315
+ train_file:
316
+ desc: null
317
+ value: /dev/shm/data-files
318
+ train_ref_file:
319
+ desc: null
320
+ value: null
321
+ use_fast_tokenizer:
322
+ desc: null
323
+ value: true
324
+ use_legacy_prediction_loop:
325
+ desc: null
326
+ value: false
327
+ validation_file:
328
+ desc: null
329
+ value: .
330
+ validation_ref_file:
331
+ desc: null
332
+ value: null
333
+ validation_split_percentage:
334
+ desc: null
335
+ value: 5
336
+ warmup_ratio:
337
+ desc: null
338
+ value: 0.0
339
+ warmup_steps:
340
+ desc: null
341
+ value: 0
342
+ weight_decay:
343
+ desc: null
344
+ value: 0.01
345
+ xpu_backend:
346
+ desc: null
347
+ value: null
wandb/run-20211108_080537-1gq01aek/files/output.log ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ [08:06:04] - INFO - absl - A polynomial schedule was set with a non-positive `transition_steps` value; this results in a constant schedule with value `init_value`.
3
+ [08:06:04] - INFO - absl - Restoring checkpoint from checkpoint_100000
4
+ tcmalloc: large alloc 2886115328 bytes == 0x943cc000 @ 0x7f99f07a3680 0x7f99f07c4824 0x5bc8a7 0x50a83b 0x50c08b 0x5f68ef 0x57107c 0x56a0ba 0x5f6343 0x56bddd 0x56a0ba 0x68d5b7 0x67cd01 0x67cd7f 0x67ce21 0x67ef47 0x6b7242 0x6b75cd 0x7f99f05ba0b3 0x5fb18e
5
+ Traceback (most recent call last):
6
+ File "run_muril_bb_flax_mlm.py", line 612, in <module>
7
+ state = restore_checkpoint(training_args.resume_from_checkpoint, state)
8
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/training/checkpoints.py", line 274, in restore_checkpoint
9
+ results = list(results)
10
+ File "/usr/lib/python3.8/concurrent/futures/_base.py", line 619, in result_iterator
11
+ yield fs.pop().result()
12
+ File "/usr/lib/python3.8/concurrent/futures/_base.py", line 439, in result
13
+ self._condition.wait(timeout)
14
+ File "/usr/lib/python3.8/threading.py", line 302, in wait
15
+ waiter.acquire()
16
+ KeyboardInterrupt
wandb/run-20211108_080537-1gq01aek/files/requirements.txt ADDED
@@ -0,0 +1,101 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ absl-py==0.15.0
2
+ aiohttp==3.8.0
3
+ aiosignal==1.2.0
4
+ astunparse==1.6.3
5
+ async-timeout==4.0.0
6
+ attrs==21.2.0
7
+ cachetools==4.2.4
8
+ certifi==2021.10.8
9
+ charset-normalizer==2.0.7
10
+ chex==0.0.8
11
+ click==8.0.3
12
+ configparser==5.1.0
13
+ cycler==0.11.0
14
+ datasets==1.15.1
15
+ dill==0.3.4
16
+ dm-tree==0.1.6
17
+ docker-pycreds==0.4.0
18
+ filelock==3.3.2
19
+ flatbuffers==2.0
20
+ flax==0.3.6
21
+ frozenlist==1.2.0
22
+ fsspec==2021.11.0
23
+ gast==0.4.0
24
+ gitdb==4.0.9
25
+ gitpython==3.1.24
26
+ google-auth-oauthlib==0.4.6
27
+ google-auth==2.3.3
28
+ google-pasta==0.2.0
29
+ grpcio==1.41.1
30
+ h5py==3.5.0
31
+ huggingface-hub==0.1.1
32
+ idna==3.3
33
+ jax==0.2.24
34
+ jaxlib==0.1.73
35
+ joblib==1.1.0
36
+ kaggle==1.5.12
37
+ keras-preprocessing==1.1.2
38
+ keras==2.7.0
39
+ kiwisolver==1.3.2
40
+ libclang==12.0.0
41
+ libtpu-nightly==0.1.dev20211018
42
+ markdown==3.3.4
43
+ matplotlib==3.4.3
44
+ msgpack==1.0.2
45
+ multidict==5.2.0
46
+ multiprocess==0.70.12.2
47
+ numpy==1.21.4
48
+ oauthlib==3.1.1
49
+ opt-einsum==3.3.0
50
+ optax==0.0.9
51
+ packaging==21.2
52
+ pandas==1.3.4
53
+ pathtools==0.1.2
54
+ pillow==8.4.0
55
+ pip==21.3.1
56
+ pkg-resources==0.0.0
57
+ promise==2.3
58
+ protobuf==3.19.1
59
+ psutil==5.8.0
60
+ pyarrow==6.0.0
61
+ pyasn1-modules==0.2.8
62
+ pyasn1==0.4.8
63
+ pyparsing==2.4.7
64
+ python-dateutil==2.8.2
65
+ python-slugify==5.0.2
66
+ pytz==2021.3
67
+ pyyaml==6.0
68
+ regex==2021.11.2
69
+ requests-oauthlib==1.3.0
70
+ requests==2.26.0
71
+ rsa==4.7.2
72
+ sacremoses==0.0.46
73
+ scipy==1.7.1
74
+ sentry-sdk==1.4.3
75
+ setuptools==44.0.0
76
+ shortuuid==1.0.1
77
+ six==1.16.0
78
+ smmap==5.0.0
79
+ subprocess32==3.5.4
80
+ tensorboard-data-server==0.6.1
81
+ tensorboard-plugin-wit==1.8.0
82
+ tensorboard==2.7.0
83
+ tensorflow-estimator==2.7.0
84
+ tensorflow-io-gcs-filesystem==0.21.0
85
+ tensorflow==2.7.0
86
+ termcolor==1.1.0
87
+ text-unidecode==1.3
88
+ tokenizers==0.10.3
89
+ toolz==0.11.1
90
+ torch==1.10.0
91
+ tqdm==4.62.3
92
+ transformers==4.13.0.dev0
93
+ typing-extensions==3.10.0.2
94
+ urllib3==1.26.7
95
+ wandb==0.12.6
96
+ werkzeug==2.0.2
97
+ wheel==0.37.0
98
+ wrapt==1.13.3
99
+ xxhash==2.0.2
100
+ yarl==1.7.2
101
+ yaspin==2.1.0
wandb/run-20211108_080537-1gq01aek/files/wandb-metadata.json ADDED
@@ -0,0 +1,69 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29",
3
+ "python": "3.8.10",
4
+ "heartbeatAt": "2021-11-08T08:05:40.298252",
5
+ "startedAt": "2021-11-08T08:05:37.281308",
6
+ "docker": null,
7
+ "cpu_count": 96,
8
+ "cuda": null,
9
+ "args": [
10
+ "--model_name_or_path",
11
+ "muril-bigbird",
12
+ "--tokenizer_name",
13
+ "muril-bigbird",
14
+ "--dtype",
15
+ "bfloat16",
16
+ "--pretokenized",
17
+ "--resume_from_checkpoint",
18
+ "checkpoint_100000",
19
+ "--train_file",
20
+ "/dev/shm/data-files",
21
+ "--validation_file",
22
+ ".",
23
+ "--max_seq_length",
24
+ "4096",
25
+ "--pad_to_max_length",
26
+ "--output_dir",
27
+ "./",
28
+ "--overwrite_output_dir",
29
+ "--do_train",
30
+ "--do_eval",
31
+ "--eval_steps",
32
+ "5000",
33
+ "--per_device_train_batch_size",
34
+ "1",
35
+ "--per_device_eval_batch_size",
36
+ "2",
37
+ "--learning_rate",
38
+ "5e-5",
39
+ "--weight_decay",
40
+ "0.01",
41
+ "--num_train_epochs",
42
+ "3",
43
+ "--warmup_steps",
44
+ "0",
45
+ "--logging_steps",
46
+ "1000",
47
+ "--save_steps",
48
+ "25000",
49
+ "--save_total_limit",
50
+ "1",
51
+ "--seed",
52
+ "18",
53
+ "--push_to_hub",
54
+ "--run_name",
55
+ "en-ta-hi-te-from-100k"
56
+ ],
57
+ "state": "running",
58
+ "program": "run_muril_bb_flax_mlm.py",
59
+ "codePath": "run_muril_bb_flax_mlm.py",
60
+ "git": {
61
+ "remote": "https://huggingface.co/nbroad/muril-bigbird-base-1mil",
62
+ "commit": "5fca06e3f8deb06ee5f938513b5f093fc3b66a62"
63
+ },
64
+ "email": "nicholas@nmbroad.com",
65
+ "root": "/home/nicholasbroad/muril-bigbird-base-1mil",
66
+ "host": "t1v-n-ba7aa1a5-w-0",
67
+ "username": "nicholasbroad",
68
+ "executable": "/home/nicholasbroad/venv/bin/python"
69
+ }
wandb/run-20211108_080537-1gq01aek/files/wandb-summary.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"_wandb": {"runtime": 86}}
wandb/run-20211108_080537-1gq01aek/logs/debug-internal.log ADDED
@@ -0,0 +1,159 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2021-11-08 08:05:37,994 INFO MainThread:5221 [internal.py:wandb_internal():87] W&B internal server running at pid: 5221, started at: 2021-11-08 08:05:37.993832
2
+ 2021-11-08 08:05:37,995 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: check_version
3
+ 2021-11-08 08:05:37,995 INFO WriterThread:5221 [datastore.py:open_for_write():77] open: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/run-1gq01aek.wandb
4
+ 2021-11-08 08:05:37,997 DEBUG SenderThread:5221 [sender.py:send():238] send: header
5
+ 2021-11-08 08:05:37,997 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: check_version
6
+ 2021-11-08 08:05:38,035 DEBUG SenderThread:5221 [sender.py:send():238] send: run
7
+ 2021-11-08 08:05:38,200 INFO SenderThread:5221 [dir_watcher.py:__init__():169] watching files in: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files
8
+ 2021-11-08 08:05:38,200 INFO SenderThread:5221 [sender.py:_start_run_threads():825] run started: 1gq01aek with start time 1636358737
9
+ 2021-11-08 08:05:38,200 DEBUG SenderThread:5221 [sender.py:send():238] send: summary
10
+ 2021-11-08 08:05:38,200 INFO SenderThread:5221 [sender.py:_save_file():960] saving file wandb-summary.json with policy end
11
+ 2021-11-08 08:05:38,201 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: run_start
12
+ 2021-11-08 08:05:39,203 INFO Thread-8 :5221 [dir_watcher.py:_on_file_created():217] file/dir created: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files/wandb-summary.json
13
+ 2021-11-08 08:05:40,298 DEBUG HandlerThread:5221 [meta.py:__init__():40] meta init
14
+ 2021-11-08 08:05:40,298 DEBUG HandlerThread:5221 [meta.py:__init__():54] meta init done
15
+ 2021-11-08 08:05:40,298 DEBUG HandlerThread:5221 [meta.py:probe():214] probe
16
+ 2021-11-08 08:05:40,299 DEBUG HandlerThread:5221 [meta.py:_setup_git():204] setup git
17
+ 2021-11-08 08:05:40,327 DEBUG HandlerThread:5221 [meta.py:_setup_git():211] setup git done
18
+ 2021-11-08 08:05:40,328 DEBUG HandlerThread:5221 [meta.py:_save_pip():58] save pip
19
+ 2021-11-08 08:05:40,328 DEBUG HandlerThread:5221 [meta.py:_save_pip():72] save pip done
20
+ 2021-11-08 08:05:40,328 DEBUG HandlerThread:5221 [meta.py:probe():252] probe done
21
+ 2021-11-08 08:05:40,331 DEBUG SenderThread:5221 [sender.py:send():238] send: files
22
+ 2021-11-08 08:05:40,331 INFO SenderThread:5221 [sender.py:_save_file():960] saving file wandb-metadata.json with policy now
23
+ 2021-11-08 08:05:40,337 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: stop_status
24
+ 2021-11-08 08:05:40,337 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: stop_status
25
+ 2021-11-08 08:05:40,465 DEBUG SenderThread:5221 [sender.py:send():238] send: config
26
+ 2021-11-08 08:05:40,465 DEBUG SenderThread:5221 [sender.py:send():238] send: config
27
+ 2021-11-08 08:05:40,465 DEBUG SenderThread:5221 [sender.py:send():238] send: config
28
+ 2021-11-08 08:05:40,880 INFO Thread-11 :5221 [upload_job.py:push():137] Uploaded file /tmp/tmpzyu3dzxewandb/1hif9l1i-wandb-metadata.json
29
+ 2021-11-08 08:05:41,208 INFO Thread-8 :5221 [dir_watcher.py:_on_file_created():217] file/dir created: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files/wandb-metadata.json
30
+ 2021-11-08 08:05:41,208 INFO Thread-8 :5221 [dir_watcher.py:_on_file_created():217] file/dir created: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files/output.log
31
+ 2021-11-08 08:05:41,208 INFO Thread-8 :5221 [dir_watcher.py:_on_file_created():217] file/dir created: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files/requirements.txt
32
+ 2021-11-08 08:05:55,467 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: stop_status
33
+ 2021-11-08 08:05:55,468 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: stop_status
34
+ 2021-11-08 08:06:07,218 INFO Thread-8 :5221 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files/output.log
35
+ 2021-11-08 08:06:08,375 DEBUG SenderThread:5221 [sender.py:send():238] send: stats
36
+ 2021-11-08 08:06:09,219 INFO Thread-8 :5221 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files/output.log
37
+ 2021-11-08 08:06:09,484 INFO Thread-8 :5221 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files/config.yaml
38
+ 2021-11-08 08:06:10,601 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: stop_status
39
+ 2021-11-08 08:06:11,015 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: stop_status
40
+ 2021-11-08 08:06:26,152 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: stop_status
41
+ 2021-11-08 08:06:26,152 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: stop_status
42
+ 2021-11-08 08:06:38,443 DEBUG SenderThread:5221 [sender.py:send():238] send: stats
43
+ 2021-11-08 08:06:41,289 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: stop_status
44
+ 2021-11-08 08:06:41,289 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: stop_status
45
+ 2021-11-08 08:06:56,424 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: stop_status
46
+ 2021-11-08 08:07:01,107 WARNING MainThread:5221 [internal.py:wandb_internal():146] Internal process interrupt: 1
47
+ 2021-11-08 08:07:01,624 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: stop_status
48
+ 2021-11-08 08:07:04,878 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: poll_exit
49
+ 2021-11-08 08:07:04,878 DEBUG SenderThread:5221 [sender.py:send():238] send: telemetry
50
+ 2021-11-08 08:07:04,878 DEBUG SenderThread:5221 [sender.py:send():238] send: exit
51
+ 2021-11-08 08:07:04,878 INFO SenderThread:5221 [sender.py:send_exit():370] handling exit code: 255
52
+ 2021-11-08 08:07:04,879 INFO SenderThread:5221 [sender.py:send_exit():372] handling runtime: 86
53
+ 2021-11-08 08:07:05,244 INFO Thread-8 :5221 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files/wandb-summary.json
54
+ 2021-11-08 08:07:05,244 INFO Thread-8 :5221 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files/output.log
55
+ 2021-11-08 08:07:06,223 INFO SenderThread:5221 [sender.py:_save_file():960] saving file wandb-summary.json with policy end
56
+ 2021-11-08 08:07:07,292 INFO SenderThread:5221 [sender.py:send_exit():382] send defer
57
+ 2021-11-08 08:07:07,292 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: poll_exit
58
+ 2021-11-08 08:07:07,293 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: defer
59
+ 2021-11-08 08:07:07,294 INFO HandlerThread:5221 [handler.py:handle_request_defer():147] handle defer: 0
60
+ 2021-11-08 08:07:07,294 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: defer
61
+ 2021-11-08 08:07:07,294 INFO SenderThread:5221 [sender.py:send_request_defer():391] handle sender defer: 0
62
+ 2021-11-08 08:07:07,294 INFO SenderThread:5221 [sender.py:transition_state():395] send defer: 1
63
+ 2021-11-08 08:07:07,294 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: defer
64
+ 2021-11-08 08:07:07,294 INFO HandlerThread:5221 [handler.py:handle_request_defer():147] handle defer: 1
65
+ 2021-11-08 08:07:07,312 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: defer
66
+ 2021-11-08 08:07:07,312 INFO SenderThread:5221 [sender.py:send_request_defer():391] handle sender defer: 1
67
+ 2021-11-08 08:07:07,312 INFO SenderThread:5221 [sender.py:transition_state():395] send defer: 2
68
+ 2021-11-08 08:07:07,312 DEBUG SenderThread:5221 [sender.py:send():238] send: stats
69
+ 2021-11-08 08:07:07,312 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: defer
70
+ 2021-11-08 08:07:07,312 INFO HandlerThread:5221 [handler.py:handle_request_defer():147] handle defer: 2
71
+ 2021-11-08 08:07:07,313 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: defer
72
+ 2021-11-08 08:07:07,313 INFO SenderThread:5221 [sender.py:send_request_defer():391] handle sender defer: 2
73
+ 2021-11-08 08:07:07,313 INFO SenderThread:5221 [sender.py:transition_state():395] send defer: 3
74
+ 2021-11-08 08:07:07,313 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: defer
75
+ 2021-11-08 08:07:07,313 INFO HandlerThread:5221 [handler.py:handle_request_defer():147] handle defer: 3
76
+ 2021-11-08 08:07:07,313 DEBUG SenderThread:5221 [sender.py:send():238] send: summary
77
+ 2021-11-08 08:07:07,396 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: poll_exit
78
+ 2021-11-08 08:07:08,008 INFO SenderThread:5221 [sender.py:_save_file():960] saving file wandb-summary.json with policy end
79
+ 2021-11-08 08:07:08,008 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: defer
80
+ 2021-11-08 08:07:08,009 INFO SenderThread:5221 [sender.py:send_request_defer():391] handle sender defer: 3
81
+ 2021-11-08 08:07:08,009 INFO SenderThread:5221 [sender.py:transition_state():395] send defer: 4
82
+ 2021-11-08 08:07:08,009 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: poll_exit
83
+ 2021-11-08 08:07:08,009 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: defer
84
+ 2021-11-08 08:07:08,010 INFO HandlerThread:5221 [handler.py:handle_request_defer():147] handle defer: 4
85
+ 2021-11-08 08:07:08,010 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: defer
86
+ 2021-11-08 08:07:08,010 INFO SenderThread:5221 [sender.py:send_request_defer():391] handle sender defer: 4
87
+ 2021-11-08 08:07:08,111 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: poll_exit
88
+ 2021-11-08 08:07:08,245 INFO Thread-8 :5221 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files/wandb-summary.json
89
+ 2021-11-08 08:07:08,245 INFO Thread-8 :5221 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files/config.yaml
90
+ 2021-11-08 08:07:09,067 INFO SenderThread:5221 [sender.py:transition_state():395] send defer: 5
91
+ 2021-11-08 08:07:10,121 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: poll_exit
92
+ 2021-11-08 08:07:10,122 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: defer
93
+ 2021-11-08 08:07:10,122 INFO HandlerThread:5221 [handler.py:handle_request_defer():147] handle defer: 5
94
+ 2021-11-08 08:07:10,123 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: defer
95
+ 2021-11-08 08:07:10,123 INFO SenderThread:5221 [sender.py:send_request_defer():391] handle sender defer: 5
96
+ 2021-11-08 08:07:10,123 INFO SenderThread:5221 [dir_watcher.py:finish():283] shutting down directory watcher
97
+ 2021-11-08 08:07:10,224 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: poll_exit
98
+ 2021-11-08 08:07:10,246 INFO SenderThread:5221 [dir_watcher.py:finish():313] scan: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files
99
+ 2021-11-08 08:07:10,247 INFO SenderThread:5221 [dir_watcher.py:finish():327] scan save: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files/requirements.txt requirements.txt
100
+ 2021-11-08 08:07:10,247 INFO SenderThread:5221 [dir_watcher.py:finish():327] scan save: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files/output.log output.log
101
+ 2021-11-08 08:07:10,247 INFO SenderThread:5221 [dir_watcher.py:finish():327] scan save: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files/wandb-metadata.json wandb-metadata.json
102
+ 2021-11-08 08:07:10,247 INFO SenderThread:5221 [dir_watcher.py:finish():327] scan save: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files/config.yaml config.yaml
103
+ 2021-11-08 08:07:10,248 INFO SenderThread:5221 [dir_watcher.py:finish():327] scan save: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files/wandb-summary.json wandb-summary.json
104
+ 2021-11-08 08:07:10,248 INFO SenderThread:5221 [sender.py:transition_state():395] send defer: 6
105
+ 2021-11-08 08:07:10,248 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: poll_exit
106
+ 2021-11-08 08:07:10,252 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: defer
107
+ 2021-11-08 08:07:10,252 INFO HandlerThread:5221 [handler.py:handle_request_defer():147] handle defer: 6
108
+ 2021-11-08 08:07:10,255 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: defer
109
+ 2021-11-08 08:07:10,255 INFO SenderThread:5221 [sender.py:send_request_defer():391] handle sender defer: 6
110
+ 2021-11-08 08:07:10,255 INFO SenderThread:5221 [file_pusher.py:finish():177] shutting down file pusher
111
+ 2021-11-08 08:07:10,360 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: poll_exit
112
+ 2021-11-08 08:07:10,360 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: poll_exit
113
+ 2021-11-08 08:07:10,462 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: poll_exit
114
+ 2021-11-08 08:07:10,462 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: poll_exit
115
+ 2021-11-08 08:07:10,563 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: poll_exit
116
+ 2021-11-08 08:07:10,564 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: poll_exit
117
+ 2021-11-08 08:07:10,665 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: poll_exit
118
+ 2021-11-08 08:07:10,665 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: poll_exit
119
+ 2021-11-08 08:07:10,751 INFO Thread-14 :5221 [upload_job.py:push():137] Uploaded file /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files/config.yaml
120
+ 2021-11-08 08:07:10,756 INFO Thread-12 :5221 [upload_job.py:push():137] Uploaded file /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files/requirements.txt
121
+ 2021-11-08 08:07:10,767 INFO Thread-13 :5221 [upload_job.py:push():137] Uploaded file /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files/output.log
122
+ 2021-11-08 08:07:10,768 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: poll_exit
123
+ 2021-11-08 08:07:10,768 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: poll_exit
124
+ 2021-11-08 08:07:10,788 INFO Thread-15 :5221 [upload_job.py:push():137] Uploaded file /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files/wandb-summary.json
125
+ 2021-11-08 08:07:10,869 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: poll_exit
126
+ 2021-11-08 08:07:10,870 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: poll_exit
127
+ 2021-11-08 08:07:10,971 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: poll_exit
128
+ 2021-11-08 08:07:10,971 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: poll_exit
129
+ 2021-11-08 08:07:10,988 INFO Thread-7 :5221 [sender.py:transition_state():395] send defer: 7
130
+ 2021-11-08 08:07:10,989 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: defer
131
+ 2021-11-08 08:07:10,989 INFO HandlerThread:5221 [handler.py:handle_request_defer():147] handle defer: 7
132
+ 2021-11-08 08:07:10,989 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: defer
133
+ 2021-11-08 08:07:10,989 INFO SenderThread:5221 [sender.py:send_request_defer():391] handle sender defer: 7
134
+ 2021-11-08 08:07:11,073 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: poll_exit
135
+ 2021-11-08 08:07:11,286 INFO SenderThread:5221 [sender.py:transition_state():395] send defer: 8
136
+ 2021-11-08 08:07:11,286 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: poll_exit
137
+ 2021-11-08 08:07:11,287 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: defer
138
+ 2021-11-08 08:07:11,287 INFO HandlerThread:5221 [handler.py:handle_request_defer():147] handle defer: 8
139
+ 2021-11-08 08:07:11,287 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: defer
140
+ 2021-11-08 08:07:11,287 INFO SenderThread:5221 [sender.py:send_request_defer():391] handle sender defer: 8
141
+ 2021-11-08 08:07:11,287 INFO SenderThread:5221 [sender.py:transition_state():395] send defer: 9
142
+ 2021-11-08 08:07:11,288 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: defer
143
+ 2021-11-08 08:07:11,288 INFO HandlerThread:5221 [handler.py:handle_request_defer():147] handle defer: 9
144
+ 2021-11-08 08:07:11,288 DEBUG SenderThread:5221 [sender.py:send():238] send: final
145
+ 2021-11-08 08:07:11,288 DEBUG SenderThread:5221 [sender.py:send():238] send: footer
146
+ 2021-11-08 08:07:11,288 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: defer
147
+ 2021-11-08 08:07:11,288 INFO SenderThread:5221 [sender.py:send_request_defer():391] handle sender defer: 9
148
+ 2021-11-08 08:07:11,388 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: poll_exit
149
+ 2021-11-08 08:07:11,388 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: poll_exit
150
+ 2021-11-08 08:07:11,389 INFO SenderThread:5221 [file_pusher.py:join():182] waiting for file pusher
151
+ 2021-11-08 08:07:12,169 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: get_summary
152
+ 2021-11-08 08:07:12,170 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: sampled_history
153
+ 2021-11-08 08:07:12,170 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: shutdown
154
+ 2021-11-08 08:07:12,171 INFO HandlerThread:5221 [handler.py:finish():731] shutting down handler
155
+ 2021-11-08 08:07:12,288 INFO WriterThread:5221 [datastore.py:close():281] close: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/run-1gq01aek.wandb
156
+ 2021-11-08 08:07:12,652 INFO SenderThread:5221 [sender.py:finish():1091] shutting down sender
157
+ 2021-11-08 08:07:12,652 INFO SenderThread:5221 [file_pusher.py:finish():177] shutting down file pusher
158
+ 2021-11-08 08:07:12,652 INFO SenderThread:5221 [file_pusher.py:join():182] waiting for file pusher
159
+ 2021-11-08 08:07:12,654 INFO MainThread:5221 [internal.py:handle_exit():77] Internal process exited
wandb/run-20211108_080537-1gq01aek/logs/debug.log ADDED
@@ -0,0 +1,137 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2021-11-08 08:05:37,301 INFO MainThread:3899 [wandb_setup.py:_flush():71] setting env: {}
2
+ 2021-11-08 08:05:37,301 INFO MainThread:3899 [wandb_setup.py:_flush():71] setting login settings: {}
3
+ 2021-11-08 08:05:37,301 INFO MainThread:3899 [wandb_init.py:_log_setup():357] Logging user logs to /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/logs/debug.log
4
+ 2021-11-08 08:05:37,301 INFO MainThread:3899 [wandb_init.py:_log_setup():358] Logging internal logs to /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/logs/debug-internal.log
5
+ 2021-11-08 08:05:37,301 INFO MainThread:3899 [wandb_init.py:init():390] calling init triggers
6
+ 2021-11-08 08:05:37,301 INFO MainThread:3899 [wandb_init.py:init():395] wandb.init called with sweep_config: {}
7
+ config: {}
8
+ 2021-11-08 08:05:37,301 INFO MainThread:3899 [wandb_init.py:init():435] starting backend
9
+ 2021-11-08 08:05:37,301 INFO MainThread:3899 [backend.py:_multiprocessing_setup():95] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
10
+ 2021-11-08 08:05:37,369 INFO MainThread:3899 [backend.py:ensure_launched():199] starting backend process...
11
+ 2021-11-08 08:05:37,422 INFO MainThread:3899 [backend.py:ensure_launched():204] started backend process with pid: 5221
12
+ 2021-11-08 08:05:37,424 INFO MainThread:3899 [wandb_init.py:init():444] backend started and connected
13
+ 2021-11-08 08:05:37,427 INFO MainThread:3899 [wandb_init.py:init():503] updated telemetry
14
+ 2021-11-08 08:05:37,515 INFO MainThread:3899 [wandb_init.py:init():533] communicating current version
15
+ 2021-11-08 08:05:38,033 INFO MainThread:3899 [wandb_init.py:init():538] got version response
16
+ 2021-11-08 08:05:38,034 INFO MainThread:3899 [wandb_init.py:init():548] communicating run to backend with 30 second timeout
17
+ 2021-11-08 08:05:38,200 INFO MainThread:3899 [wandb_init.py:init():576] starting run threads in backend
18
+ 2021-11-08 08:05:40,336 INFO MainThread:3899 [wandb_run.py:_console_start():1693] atexit reg
19
+ 2021-11-08 08:05:40,337 INFO MainThread:3899 [wandb_run.py:_redirect():1567] redirect: SettingsConsole.REDIRECT
20
+ 2021-11-08 08:05:40,337 INFO MainThread:3899 [wandb_run.py:_redirect():1572] Redirecting console.
21
+ 2021-11-08 08:05:40,339 INFO MainThread:3899 [wandb_run.py:_redirect():1628] Redirects installed.
22
+ 2021-11-08 08:05:40,339 INFO MainThread:3899 [wandb_init.py:init():603] run started, returning control to user process
23
+ 2021-11-08 08:05:40,345 INFO MainThread:3899 [wandb_run.py:_config_callback():911] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': True, 'do_eval': True, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 2, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 5e-05, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 3.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 0, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Nov08_08-04-38_t1v-n-ba7aa1a5-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 1000, 'logging_nan_inf_filter': True, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 25000, 'save_total_limit': 1, 'save_on_each_node': False, 'no_cuda': False, 'seed': 18, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'xpu_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 5000, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': 'en-ta-hi-te-from-100k', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': 'checkpoint_100000', 'hub_model_id': None, 'hub_strategy': 'HubStrategy.EVERY_SAVE', 'hub_token': None, 'gradient_checkpointing': False, 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': '', '_n_gpu': 0, '__cached__setup_devices': 'cpu'}
24
+ 2021-11-08 08:05:40,347 INFO MainThread:3899 [wandb_run.py:_config_callback():911] config_cb None None {'model_name_or_path': 'muril-bigbird', 'model_type': None, 'config_name': None, 'tokenizer_name': 'muril-bigbird', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'bfloat16'}
25
+ 2021-11-08 08:05:40,348 INFO MainThread:3899 [wandb_run.py:_config_callback():911] config_cb None None {'dataset_name': None, 'pretokenized': True, 'streaming': False, 'dataset_config_name': None, 'train_file': '/dev/shm/data-files', 'validation_file': '.', 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': None, 'mlm_probability': 0.15, 'pad_to_max_length': True, 'line_by_line': False}
26
+ 2021-11-08 08:07:02,826 INFO MainThread:3899 [wandb_run.py:_atexit_cleanup():1663] got exitcode: 255
27
+ 2021-11-08 08:07:02,826 INFO MainThread:3899 [wandb_run.py:_restore():1635] restore
28
+ 2021-11-08 08:07:07,294 INFO MainThread:3899 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
29
+ wandb_count: 1
30
+ }
31
+ pusher_stats {
32
+ uploaded_bytes: 1825
33
+ total_bytes: 1825
34
+ }
35
+
36
+ 2021-11-08 08:07:08,010 INFO MainThread:3899 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
37
+ wandb_count: 1
38
+ }
39
+ pusher_stats {
40
+ uploaded_bytes: 1825
41
+ total_bytes: 1825
42
+ }
43
+
44
+ 2021-11-08 08:07:10,122 INFO MainThread:3899 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
45
+ wandb_count: 1
46
+ }
47
+ pusher_stats {
48
+ uploaded_bytes: 1825
49
+ total_bytes: 1825
50
+ }
51
+
52
+ 2021-11-08 08:07:10,259 INFO MainThread:3899 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
53
+ wandb_count: 3
54
+ }
55
+ pusher_stats {
56
+ uploaded_bytes: 1825
57
+ total_bytes: 4717
58
+ }
59
+
60
+ 2021-11-08 08:07:10,361 INFO MainThread:3899 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
61
+ wandb_count: 5
62
+ }
63
+ pusher_stats {
64
+ uploaded_bytes: 1825
65
+ total_bytes: 9972
66
+ }
67
+
68
+ 2021-11-08 08:07:10,462 INFO MainThread:3899 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
69
+ wandb_count: 5
70
+ }
71
+ pusher_stats {
72
+ uploaded_bytes: 9972
73
+ total_bytes: 9972
74
+ }
75
+
76
+ 2021-11-08 08:07:10,564 INFO MainThread:3899 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
77
+ wandb_count: 5
78
+ }
79
+ pusher_stats {
80
+ uploaded_bytes: 9972
81
+ total_bytes: 9972
82
+ }
83
+
84
+ 2021-11-08 08:07:10,666 INFO MainThread:3899 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
85
+ wandb_count: 5
86
+ }
87
+ pusher_stats {
88
+ uploaded_bytes: 9972
89
+ total_bytes: 9972
90
+ }
91
+
92
+ 2021-11-08 08:07:10,769 INFO MainThread:3899 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
93
+ wandb_count: 5
94
+ }
95
+ pusher_stats {
96
+ uploaded_bytes: 9972
97
+ total_bytes: 9972
98
+ }
99
+
100
+ 2021-11-08 08:07:10,870 INFO MainThread:3899 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
101
+ wandb_count: 5
102
+ }
103
+ pusher_stats {
104
+ uploaded_bytes: 9972
105
+ total_bytes: 9972
106
+ }
107
+
108
+ 2021-11-08 08:07:10,972 INFO MainThread:3899 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
109
+ wandb_count: 5
110
+ }
111
+ pusher_stats {
112
+ uploaded_bytes: 9972
113
+ total_bytes: 9972
114
+ }
115
+
116
+ 2021-11-08 08:07:11,287 INFO MainThread:3899 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
117
+ wandb_count: 5
118
+ }
119
+ pusher_stats {
120
+ uploaded_bytes: 9972
121
+ total_bytes: 9972
122
+ }
123
+
124
+ 2021-11-08 08:07:11,652 INFO MainThread:3899 [wandb_run.py:_wait_for_finish():1793] got exit ret: done: true
125
+ exit_result {
126
+ }
127
+ file_counts {
128
+ wandb_count: 5
129
+ }
130
+ pusher_stats {
131
+ uploaded_bytes: 9972
132
+ total_bytes: 9972
133
+ }
134
+ local_info {
135
+ }
136
+
137
+ 2021-11-08 08:07:33,406 INFO MainThread:3899 [wandb_run.py:_append_files():2061] logging synced files
wandb/run-20211108_080537-1gq01aek/run-1gq01aek.wandb ADDED
Binary file (5.36 kB). View file
 
wandb/run-20211108_080817-3dz8poa1/files/config.yaml ADDED
@@ -0,0 +1,347 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ wandb_version: 1
2
+
3
+ __cached__setup_devices:
4
+ desc: null
5
+ value: cpu
6
+ _n_gpu:
7
+ desc: null
8
+ value: 0
9
+ _wandb:
10
+ desc: null
11
+ value:
12
+ cli_version: 0.12.6
13
+ framework: huggingface
14
+ huggingface_version: 4.13.0.dev0
15
+ is_jupyter_run: false
16
+ is_kaggle_kernel: true
17
+ python_version: 3.8.10
18
+ start_time: 1636358897
19
+ t:
20
+ 1:
21
+ - 1
22
+ - 2
23
+ - 3
24
+ - 11
25
+ - 12
26
+ 2:
27
+ - 1
28
+ - 2
29
+ - 3
30
+ - 11
31
+ - 12
32
+ 3:
33
+ - 13
34
+ 4: 3.8.10
35
+ 5: 0.12.6
36
+ 6: 4.13.0.dev0
37
+ 8:
38
+ - 2
39
+ - 5
40
+ adafactor:
41
+ desc: null
42
+ value: false
43
+ adam_beta1:
44
+ desc: null
45
+ value: 0.9
46
+ adam_beta2:
47
+ desc: null
48
+ value: 0.999
49
+ adam_epsilon:
50
+ desc: null
51
+ value: 1.0e-08
52
+ cache_dir:
53
+ desc: null
54
+ value: null
55
+ config_name:
56
+ desc: null
57
+ value: null
58
+ dataloader_drop_last:
59
+ desc: null
60
+ value: false
61
+ dataloader_num_workers:
62
+ desc: null
63
+ value: 0
64
+ dataloader_pin_memory:
65
+ desc: null
66
+ value: true
67
+ dataset_config_name:
68
+ desc: null
69
+ value: null
70
+ dataset_name:
71
+ desc: null
72
+ value: null
73
+ ddp_find_unused_parameters:
74
+ desc: null
75
+ value: null
76
+ debug:
77
+ desc: null
78
+ value: []
79
+ deepspeed:
80
+ desc: null
81
+ value: null
82
+ disable_tqdm:
83
+ desc: null
84
+ value: false
85
+ do_eval:
86
+ desc: null
87
+ value: true
88
+ do_predict:
89
+ desc: null
90
+ value: false
91
+ do_train:
92
+ desc: null
93
+ value: true
94
+ dtype:
95
+ desc: null
96
+ value: bfloat16
97
+ eval_accumulation_steps:
98
+ desc: null
99
+ value: null
100
+ eval_steps:
101
+ desc: null
102
+ value: 5000
103
+ evaluation_strategy:
104
+ desc: null
105
+ value: IntervalStrategy.NO
106
+ fp16:
107
+ desc: null
108
+ value: false
109
+ fp16_backend:
110
+ desc: null
111
+ value: auto
112
+ fp16_full_eval:
113
+ desc: null
114
+ value: false
115
+ fp16_opt_level:
116
+ desc: null
117
+ value: O1
118
+ gradient_accumulation_steps:
119
+ desc: null
120
+ value: 1
121
+ gradient_checkpointing:
122
+ desc: null
123
+ value: false
124
+ greater_is_better:
125
+ desc: null
126
+ value: null
127
+ group_by_length:
128
+ desc: null
129
+ value: false
130
+ hub_model_id:
131
+ desc: null
132
+ value: null
133
+ hub_strategy:
134
+ desc: null
135
+ value: HubStrategy.EVERY_SAVE
136
+ hub_token:
137
+ desc: null
138
+ value: null
139
+ ignore_data_skip:
140
+ desc: null
141
+ value: false
142
+ label_names:
143
+ desc: null
144
+ value: null
145
+ label_smoothing_factor:
146
+ desc: null
147
+ value: 0.0
148
+ learning_rate:
149
+ desc: null
150
+ value: 5.0e-05
151
+ length_column_name:
152
+ desc: null
153
+ value: length
154
+ line_by_line:
155
+ desc: null
156
+ value: false
157
+ load_best_model_at_end:
158
+ desc: null
159
+ value: false
160
+ local_rank:
161
+ desc: null
162
+ value: -1
163
+ log_level:
164
+ desc: null
165
+ value: -1
166
+ log_level_replica:
167
+ desc: null
168
+ value: -1
169
+ log_on_each_node:
170
+ desc: null
171
+ value: true
172
+ logging_dir:
173
+ desc: null
174
+ value: ./runs/Nov08_08-08-10_t1v-n-ba7aa1a5-w-0
175
+ logging_first_step:
176
+ desc: null
177
+ value: false
178
+ logging_nan_inf_filter:
179
+ desc: null
180
+ value: true
181
+ logging_steps:
182
+ desc: null
183
+ value: 1000
184
+ logging_strategy:
185
+ desc: null
186
+ value: IntervalStrategy.STEPS
187
+ lr_scheduler_type:
188
+ desc: null
189
+ value: SchedulerType.LINEAR
190
+ max_grad_norm:
191
+ desc: null
192
+ value: 1.0
193
+ max_seq_length:
194
+ desc: null
195
+ value: 4096
196
+ max_steps:
197
+ desc: null
198
+ value: -1
199
+ metric_for_best_model:
200
+ desc: null
201
+ value: null
202
+ mlm_probability:
203
+ desc: null
204
+ value: 0.15
205
+ model_name_or_path:
206
+ desc: null
207
+ value: muril-bigbird
208
+ model_type:
209
+ desc: null
210
+ value: null
211
+ mp_parameters:
212
+ desc: null
213
+ value: ''
214
+ no_cuda:
215
+ desc: null
216
+ value: false
217
+ num_train_epochs:
218
+ desc: null
219
+ value: 3.0
220
+ output_dir:
221
+ desc: null
222
+ value: ./
223
+ overwrite_cache:
224
+ desc: null
225
+ value: false
226
+ overwrite_output_dir:
227
+ desc: null
228
+ value: true
229
+ pad_to_max_length:
230
+ desc: null
231
+ value: true
232
+ past_index:
233
+ desc: null
234
+ value: -1
235
+ per_device_eval_batch_size:
236
+ desc: null
237
+ value: 2
238
+ per_device_train_batch_size:
239
+ desc: null
240
+ value: 1
241
+ per_gpu_eval_batch_size:
242
+ desc: null
243
+ value: null
244
+ per_gpu_train_batch_size:
245
+ desc: null
246
+ value: null
247
+ prediction_loss_only:
248
+ desc: null
249
+ value: false
250
+ preprocessing_num_workers:
251
+ desc: null
252
+ value: null
253
+ pretokenized:
254
+ desc: null
255
+ value: true
256
+ push_to_hub:
257
+ desc: null
258
+ value: true
259
+ push_to_hub_model_id:
260
+ desc: null
261
+ value: null
262
+ push_to_hub_organization:
263
+ desc: null
264
+ value: null
265
+ push_to_hub_token:
266
+ desc: null
267
+ value: null
268
+ remove_unused_columns:
269
+ desc: null
270
+ value: true
271
+ report_to:
272
+ desc: null
273
+ value:
274
+ - tensorboard
275
+ - wandb
276
+ resume_from_checkpoint:
277
+ desc: null
278
+ value: checkpoint_100000
279
+ run_name:
280
+ desc: null
281
+ value: en-ta-hi-te-from-100k
282
+ save_on_each_node:
283
+ desc: null
284
+ value: false
285
+ save_steps:
286
+ desc: null
287
+ value: 25000
288
+ save_strategy:
289
+ desc: null
290
+ value: IntervalStrategy.STEPS
291
+ save_total_limit:
292
+ desc: null
293
+ value: 1
294
+ seed:
295
+ desc: null
296
+ value: 18
297
+ sharded_ddp:
298
+ desc: null
299
+ value: []
300
+ skip_memory_metrics:
301
+ desc: null
302
+ value: true
303
+ streaming:
304
+ desc: null
305
+ value: false
306
+ tokenizer_name:
307
+ desc: null
308
+ value: muril-bigbird
309
+ tpu_metrics_debug:
310
+ desc: null
311
+ value: false
312
+ tpu_num_cores:
313
+ desc: null
314
+ value: null
315
+ train_file:
316
+ desc: null
317
+ value: /dev/shm/data-files
318
+ train_ref_file:
319
+ desc: null
320
+ value: null
321
+ use_fast_tokenizer:
322
+ desc: null
323
+ value: true
324
+ use_legacy_prediction_loop:
325
+ desc: null
326
+ value: false
327
+ validation_file:
328
+ desc: null
329
+ value: .
330
+ validation_ref_file:
331
+ desc: null
332
+ value: null
333
+ validation_split_percentage:
334
+ desc: null
335
+ value: 5
336
+ warmup_ratio:
337
+ desc: null
338
+ value: 0.0
339
+ warmup_steps:
340
+ desc: null
341
+ value: 0
342
+ weight_decay:
343
+ desc: null
344
+ value: 0.01
345
+ xpu_backend:
346
+ desc: null
347
+ value: null
wandb/run-20211108_080817-3dz8poa1/files/output.log ADDED
@@ -0,0 +1,120 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Traceback (most recent call last):
2
+ File "run_muril_bb_flax_mlm.py", line 548, in <module>
3
+ model = FlaxAutoModelForMaskedLM.from_pretrained(
4
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/transformers/models/auto/auto_factory.py", line 419, in from_pretrained
5
+ return model_class.from_pretrained(pretrained_model_name_or_path, *model_args, config=config, **kwargs)
6
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/transformers/modeling_flax_utils.py", line 351, in from_pretrained
7
+ model = cls(config, *model_args, **model_kwargs)
8
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/transformers/models/big_bird/modeling_flax_big_bird.py", line 1359, in __init__
9
+ super().__init__(config, module, input_shape=input_shape, seed=seed, dtype=dtype)
10
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/transformers/modeling_flax_utils.py", line 106, in __init__
11
+ random_params = self.init_weights(self.key, input_shape)
12
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/transformers/models/big_bird/modeling_flax_big_bird.py", line 1371, in init_weights
13
+ return self.module.init(rngs, input_ids, attention_mask, token_type_ids, position_ids, return_dict=False)[
14
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/_src/traceback_util.py", line 162, in reraise_with_filtered_traceback
15
+ return fun(*args, **kwargs)
16
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 1122, in init
17
+ _, v_out = self.init_with_output(
18
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/_src/traceback_util.py", line 162, in reraise_with_filtered_traceback
19
+ return fun(*args, **kwargs)
20
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 1091, in init_with_output
21
+ return self.apply(
22
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/_src/traceback_util.py", line 162, in reraise_with_filtered_traceback
23
+ return fun(*args, **kwargs)
24
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 1058, in apply
25
+ return apply(
26
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/core/scope.py", line 706, in wrapper
27
+ y = fn(root, *args, **kwargs)
28
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 1313, in scope_fn
29
+ return fn(module.clone(parent=scope), *args, **kwargs)
30
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/transforms.py", line 883, in wrapped_fn
31
+ return prewrapped_fn(self, *args, **kwargs)
32
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 318, in wrapped_module_method
33
+ return self._call_wrapped_method(fun, args, kwargs)
34
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 603, in _call_wrapped_method
35
+ y = fun(self, *args, **kwargs)
36
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/transformers/models/big_bird/modeling_flax_big_bird.py", line 1607, in __call__
37
+ outputs = self.bert(
38
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/transforms.py", line 883, in wrapped_fn
39
+ return prewrapped_fn(self, *args, **kwargs)
40
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 318, in wrapped_module_method
41
+ return self._call_wrapped_method(fun, args, kwargs)
42
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 603, in _call_wrapped_method
43
+ y = fun(self, *args, **kwargs)
44
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/transformers/models/big_bird/modeling_flax_big_bird.py", line 1452, in __call__
45
+ outputs = self.encoder(
46
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/transforms.py", line 883, in wrapped_fn
47
+ return prewrapped_fn(self, *args, **kwargs)
48
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 318, in wrapped_module_method
49
+ return self._call_wrapped_method(fun, args, kwargs)
50
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 603, in _call_wrapped_method
51
+ y = fun(self, *args, **kwargs)
52
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/transformers/models/big_bird/modeling_flax_big_bird.py", line 1259, in __call__
53
+ return self.layer(
54
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/transforms.py", line 883, in wrapped_fn
55
+ return prewrapped_fn(self, *args, **kwargs)
56
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 318, in wrapped_module_method
57
+ return self._call_wrapped_method(fun, args, kwargs)
58
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 603, in _call_wrapped_method
59
+ y = fun(self, *args, **kwargs)
60
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/transformers/models/big_bird/modeling_flax_big_bird.py", line 1220, in __call__
61
+ layer_outputs = layer(
62
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/transforms.py", line 883, in wrapped_fn
63
+ return prewrapped_fn(self, *args, **kwargs)
64
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 318, in wrapped_module_method
65
+ return self._call_wrapped_method(fun, args, kwargs)
66
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 603, in _call_wrapped_method
67
+ y = fun(self, *args, **kwargs)
68
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/transformers/models/big_bird/modeling_flax_big_bird.py", line 1183, in __call__
69
+ hidden_states = self.intermediate(attention_output)
70
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/transforms.py", line 883, in wrapped_fn
71
+ return prewrapped_fn(self, *args, **kwargs)
72
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 318, in wrapped_module_method
73
+ return self._call_wrapped_method(fun, args, kwargs)
74
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 603, in _call_wrapped_method
75
+ y = fun(self, *args, **kwargs)
76
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/transformers/models/big_bird/modeling_flax_big_bird.py", line 1140, in __call__
77
+ hidden_states = self.dense(hidden_states)
78
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/transforms.py", line 883, in wrapped_fn
79
+ return prewrapped_fn(self, *args, **kwargs)
80
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 318, in wrapped_module_method
81
+ return self._call_wrapped_method(fun, args, kwargs)
82
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 603, in _call_wrapped_method
83
+ y = fun(self, *args, **kwargs)
84
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/linear.py", line 171, in __call__
85
+ kernel = self.param('kernel',
86
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 896, in param
87
+ v = self.scope.param(name, init_fn, *init_args)
88
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/core/scope.py", line 643, in param
89
+ value = init_fn(self.make_rng('params'), *init_args)
90
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/_src/nn/initializers.py", line 44, in init
91
+ return random.normal(key, shape, dtype) * stddev
92
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/_src/numpy/lax_numpy.py", line 6589, in deferring_binary_op
93
+ return binary_op(self, other)
94
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/_src/traceback_util.py", line 162, in reraise_with_filtered_traceback
95
+ return fun(*args, **kwargs)
96
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/_src/api.py", line 416, in cache_miss
97
+ out_flat = xla.xla_call(
98
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/core.py", line 1632, in bind
99
+ return call_bind(self, fun, *args, **params)
100
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/core.py", line 1623, in call_bind
101
+ outs = primitive.process(top_trace, fun, tracers, params)
102
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/core.py", line 1635, in process
103
+ return trace.process_call(self, fun, tracers, params)
104
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/core.py", line 627, in process_call
105
+ return primitive.impl(f, *tracers, **params)
106
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/interpreters/xla.py", line 581, in _xla_call_impl
107
+ compiled_fun = _xla_callable(fun, device, backend, name, donated_invars,
108
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/linear_util.py", line 263, in memoized_fun
109
+ ans = call(fun, *args)
110
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/interpreters/xla.py", line 653, in _xla_callable_uncached
111
+ return lower_xla_callable(fun, device, backend, name, donated_invars,
112
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/interpreters/xla.py", line 769, in compile
113
+ self._executable = XlaCompiledComputation.from_xla_computation(
114
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/interpreters/xla.py", line 798, in from_xla_computation
115
+ compiled = compile_or_get_cached(backend, xla_computation, options)
116
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/interpreters/xla.py", line 87, in compile_or_get_cached
117
+ return backend_compile(backend, computation, compile_options)
118
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/interpreters/xla.py", line 369, in backend_compile
119
+ return backend.compile(built_c, compile_options=options)
120
+ KeyboardInterrupt
wandb/run-20211108_080817-3dz8poa1/files/requirements.txt ADDED
@@ -0,0 +1,101 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ absl-py==0.15.0
2
+ aiohttp==3.8.0
3
+ aiosignal==1.2.0
4
+ astunparse==1.6.3
5
+ async-timeout==4.0.0
6
+ attrs==21.2.0
7
+ cachetools==4.2.4
8
+ certifi==2021.10.8
9
+ charset-normalizer==2.0.7
10
+ chex==0.0.8
11
+ click==8.0.3
12
+ configparser==5.1.0
13
+ cycler==0.11.0
14
+ datasets==1.15.1
15
+ dill==0.3.4
16
+ dm-tree==0.1.6
17
+ docker-pycreds==0.4.0
18
+ filelock==3.3.2
19
+ flatbuffers==2.0
20
+ flax==0.3.6
21
+ frozenlist==1.2.0
22
+ fsspec==2021.11.0
23
+ gast==0.4.0
24
+ gitdb==4.0.9
25
+ gitpython==3.1.24
26
+ google-auth-oauthlib==0.4.6
27
+ google-auth==2.3.3
28
+ google-pasta==0.2.0
29
+ grpcio==1.41.1
30
+ h5py==3.5.0
31
+ huggingface-hub==0.1.1
32
+ idna==3.3
33
+ jax==0.2.24
34
+ jaxlib==0.1.73
35
+ joblib==1.1.0
36
+ kaggle==1.5.12
37
+ keras-preprocessing==1.1.2
38
+ keras==2.7.0
39
+ kiwisolver==1.3.2
40
+ libclang==12.0.0
41
+ libtpu-nightly==0.1.dev20211018
42
+ markdown==3.3.4
43
+ matplotlib==3.4.3
44
+ msgpack==1.0.2
45
+ multidict==5.2.0
46
+ multiprocess==0.70.12.2
47
+ numpy==1.21.4
48
+ oauthlib==3.1.1
49
+ opt-einsum==3.3.0
50
+ optax==0.0.9
51
+ packaging==21.2
52
+ pandas==1.3.4
53
+ pathtools==0.1.2
54
+ pillow==8.4.0
55
+ pip==21.3.1
56
+ pkg-resources==0.0.0
57
+ promise==2.3
58
+ protobuf==3.19.1
59
+ psutil==5.8.0
60
+ pyarrow==6.0.0
61
+ pyasn1-modules==0.2.8
62
+ pyasn1==0.4.8
63
+ pyparsing==2.4.7
64
+ python-dateutil==2.8.2
65
+ python-slugify==5.0.2
66
+ pytz==2021.3
67
+ pyyaml==6.0
68
+ regex==2021.11.2
69
+ requests-oauthlib==1.3.0
70
+ requests==2.26.0
71
+ rsa==4.7.2
72
+ sacremoses==0.0.46
73
+ scipy==1.7.1
74
+ sentry-sdk==1.4.3
75
+ setuptools==44.0.0
76
+ shortuuid==1.0.1
77
+ six==1.16.0
78
+ smmap==5.0.0
79
+ subprocess32==3.5.4
80
+ tensorboard-data-server==0.6.1
81
+ tensorboard-plugin-wit==1.8.0
82
+ tensorboard==2.7.0
83
+ tensorflow-estimator==2.7.0
84
+ tensorflow-io-gcs-filesystem==0.21.0
85
+ tensorflow==2.7.0
86
+ termcolor==1.1.0
87
+ text-unidecode==1.3
88
+ tokenizers==0.10.3
89
+ toolz==0.11.1
90
+ torch==1.10.0
91
+ tqdm==4.62.3
92
+ transformers==4.13.0.dev0
93
+ typing-extensions==3.10.0.2
94
+ urllib3==1.26.7
95
+ wandb==0.12.6
96
+ werkzeug==2.0.2
97
+ wheel==0.37.0
98
+ wrapt==1.13.3
99
+ xxhash==2.0.2
100
+ yarl==1.7.2
101
+ yaspin==2.1.0
wandb/run-20211108_080817-3dz8poa1/files/wandb-metadata.json ADDED
@@ -0,0 +1,69 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29",
3
+ "python": "3.8.10",
4
+ "heartbeatAt": "2021-11-08T08:08:20.712947",
5
+ "startedAt": "2021-11-08T08:08:17.853714",
6
+ "docker": null,
7
+ "cpu_count": 96,
8
+ "cuda": null,
9
+ "args": [
10
+ "--model_name_or_path",
11
+ "muril-bigbird",
12
+ "--tokenizer_name",
13
+ "muril-bigbird",
14
+ "--dtype",
15
+ "bfloat16",
16
+ "--pretokenized",
17
+ "--resume_from_checkpoint",
18
+ "checkpoint_100000",
19
+ "--train_file",
20
+ "/dev/shm/data-files",
21
+ "--validation_file",
22
+ ".",
23
+ "--max_seq_length",
24
+ "4096",
25
+ "--pad_to_max_length",
26
+ "--output_dir",
27
+ "./",
28
+ "--overwrite_output_dir",
29
+ "--do_train",
30
+ "--do_eval",
31
+ "--eval_steps",
32
+ "5000",
33
+ "--per_device_train_batch_size",
34
+ "1",
35
+ "--per_device_eval_batch_size",
36
+ "2",
37
+ "--learning_rate",
38
+ "5e-5",
39
+ "--weight_decay",
40
+ "0.01",
41
+ "--num_train_epochs",
42
+ "3",
43
+ "--warmup_steps",
44
+ "0",
45
+ "--logging_steps",
46
+ "1000",
47
+ "--save_steps",
48
+ "25000",
49
+ "--save_total_limit",
50
+ "1",
51
+ "--seed",
52
+ "18",
53
+ "--push_to_hub",
54
+ "--run_name",
55
+ "en-ta-hi-te-from-100k"
56
+ ],
57
+ "state": "running",
58
+ "program": "run_muril_bb_flax_mlm.py",
59
+ "codePath": "run_muril_bb_flax_mlm.py",
60
+ "git": {
61
+ "remote": "https://huggingface.co/nbroad/muril-bigbird-base-1mil",
62
+ "commit": "5fca06e3f8deb06ee5f938513b5f093fc3b66a62"
63
+ },
64
+ "email": "nicholas@nmbroad.com",
65
+ "root": "/home/nicholasbroad/muril-bigbird-base-1mil",
66
+ "host": "t1v-n-ba7aa1a5-w-0",
67
+ "username": "nicholasbroad",
68
+ "executable": "/home/nicholasbroad/venv/bin/python"
69
+ }
wandb/run-20211108_080817-3dz8poa1/files/wandb-summary.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"_wandb": {"runtime": 13}}
wandb/run-20211108_080817-3dz8poa1/logs/debug-internal.log ADDED
@@ -0,0 +1,142 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2021-11-08 08:08:18,489 INFO MainThread:6780 [internal.py:wandb_internal():87] W&B internal server running at pid: 6780, started at: 2021-11-08 08:08:18.489329
2
+ 2021-11-08 08:08:18,491 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: check_version
3
+ 2021-11-08 08:08:18,491 INFO WriterThread:6780 [datastore.py:open_for_write():77] open: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/run-3dz8poa1.wandb
4
+ 2021-11-08 08:08:18,492 DEBUG SenderThread:6780 [sender.py:send():238] send: header
5
+ 2021-11-08 08:08:18,493 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: check_version
6
+ 2021-11-08 08:08:18,528 DEBUG SenderThread:6780 [sender.py:send():238] send: run
7
+ 2021-11-08 08:08:18,688 INFO SenderThread:6780 [dir_watcher.py:__init__():169] watching files in: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/files
8
+ 2021-11-08 08:08:18,688 INFO SenderThread:6780 [sender.py:_start_run_threads():825] run started: 3dz8poa1 with start time 1636358897
9
+ 2021-11-08 08:08:18,688 DEBUG SenderThread:6780 [sender.py:send():238] send: summary
10
+ 2021-11-08 08:08:18,688 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: run_start
11
+ 2021-11-08 08:08:18,689 INFO SenderThread:6780 [sender.py:_save_file():960] saving file wandb-summary.json with policy end
12
+ 2021-11-08 08:08:19,690 INFO Thread-8 :6780 [dir_watcher.py:_on_file_created():217] file/dir created: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/files/wandb-summary.json
13
+ 2021-11-08 08:08:20,712 DEBUG HandlerThread:6780 [meta.py:__init__():40] meta init
14
+ 2021-11-08 08:08:20,712 DEBUG HandlerThread:6780 [meta.py:__init__():54] meta init done
15
+ 2021-11-08 08:08:20,712 DEBUG HandlerThread:6780 [meta.py:probe():214] probe
16
+ 2021-11-08 08:08:20,713 DEBUG HandlerThread:6780 [meta.py:_setup_git():204] setup git
17
+ 2021-11-08 08:08:20,743 DEBUG HandlerThread:6780 [meta.py:_setup_git():211] setup git done
18
+ 2021-11-08 08:08:20,743 DEBUG HandlerThread:6780 [meta.py:_save_pip():58] save pip
19
+ 2021-11-08 08:08:20,743 DEBUG HandlerThread:6780 [meta.py:_save_pip():72] save pip done
20
+ 2021-11-08 08:08:20,743 DEBUG HandlerThread:6780 [meta.py:probe():252] probe done
21
+ 2021-11-08 08:08:20,746 DEBUG SenderThread:6780 [sender.py:send():238] send: files
22
+ 2021-11-08 08:08:20,746 INFO SenderThread:6780 [sender.py:_save_file():960] saving file wandb-metadata.json with policy now
23
+ 2021-11-08 08:08:20,752 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: stop_status
24
+ 2021-11-08 08:08:20,752 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: stop_status
25
+ 2021-11-08 08:08:20,883 DEBUG SenderThread:6780 [sender.py:send():238] send: config
26
+ 2021-11-08 08:08:20,883 DEBUG SenderThread:6780 [sender.py:send():238] send: config
27
+ 2021-11-08 08:08:20,883 DEBUG SenderThread:6780 [sender.py:send():238] send: config
28
+ 2021-11-08 08:08:21,236 INFO Thread-11 :6780 [upload_job.py:push():137] Uploaded file /tmp/tmpiwrpz1kvwandb/13hbo1qc-wandb-metadata.json
29
+ 2021-11-08 08:08:21,693 INFO Thread-8 :6780 [dir_watcher.py:_on_file_created():217] file/dir created: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/files/output.log
30
+ 2021-11-08 08:08:21,693 INFO Thread-8 :6780 [dir_watcher.py:_on_file_created():217] file/dir created: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/files/wandb-metadata.json
31
+ 2021-11-08 08:08:21,693 INFO Thread-8 :6780 [dir_watcher.py:_on_file_created():217] file/dir created: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/files/requirements.txt
32
+ 2021-11-08 08:08:29,437 WARNING MainThread:6780 [internal.py:wandb_internal():146] Internal process interrupt: 1
33
+ 2021-11-08 08:08:31,697 INFO Thread-8 :6780 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/files/output.log
34
+ 2021-11-08 08:08:31,797 DEBUG SenderThread:6780 [sender.py:send():238] send: telemetry
35
+ 2021-11-08 08:08:31,797 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: poll_exit
36
+ 2021-11-08 08:08:31,797 DEBUG SenderThread:6780 [sender.py:send():238] send: exit
37
+ 2021-11-08 08:08:31,797 INFO SenderThread:6780 [sender.py:send_exit():370] handling exit code: 255
38
+ 2021-11-08 08:08:31,798 INFO SenderThread:6780 [sender.py:send_exit():372] handling runtime: 13
39
+ 2021-11-08 08:08:31,798 INFO SenderThread:6780 [sender.py:_save_file():960] saving file wandb-summary.json with policy end
40
+ 2021-11-08 08:08:31,798 INFO SenderThread:6780 [sender.py:send_exit():382] send defer
41
+ 2021-11-08 08:08:31,798 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: poll_exit
42
+ 2021-11-08 08:08:31,799 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: defer
43
+ 2021-11-08 08:08:31,799 INFO HandlerThread:6780 [handler.py:handle_request_defer():147] handle defer: 0
44
+ 2021-11-08 08:08:31,799 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: defer
45
+ 2021-11-08 08:08:31,799 INFO SenderThread:6780 [sender.py:send_request_defer():391] handle sender defer: 0
46
+ 2021-11-08 08:08:31,799 INFO SenderThread:6780 [sender.py:transition_state():395] send defer: 1
47
+ 2021-11-08 08:08:31,800 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: defer
48
+ 2021-11-08 08:08:31,800 INFO HandlerThread:6780 [handler.py:handle_request_defer():147] handle defer: 1
49
+ 2021-11-08 08:08:31,849 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: defer
50
+ 2021-11-08 08:08:31,849 INFO SenderThread:6780 [sender.py:send_request_defer():391] handle sender defer: 1
51
+ 2021-11-08 08:08:31,849 INFO SenderThread:6780 [sender.py:transition_state():395] send defer: 2
52
+ 2021-11-08 08:08:31,849 DEBUG SenderThread:6780 [sender.py:send():238] send: stats
53
+ 2021-11-08 08:08:31,850 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: defer
54
+ 2021-11-08 08:08:31,850 INFO HandlerThread:6780 [handler.py:handle_request_defer():147] handle defer: 2
55
+ 2021-11-08 08:08:31,850 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: defer
56
+ 2021-11-08 08:08:31,850 INFO SenderThread:6780 [sender.py:send_request_defer():391] handle sender defer: 2
57
+ 2021-11-08 08:08:31,850 INFO SenderThread:6780 [sender.py:transition_state():395] send defer: 3
58
+ 2021-11-08 08:08:31,850 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: defer
59
+ 2021-11-08 08:08:31,850 INFO HandlerThread:6780 [handler.py:handle_request_defer():147] handle defer: 3
60
+ 2021-11-08 08:08:31,850 DEBUG SenderThread:6780 [sender.py:send():238] send: summary
61
+ 2021-11-08 08:08:31,851 INFO SenderThread:6780 [sender.py:_save_file():960] saving file wandb-summary.json with policy end
62
+ 2021-11-08 08:08:31,851 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: defer
63
+ 2021-11-08 08:08:31,851 INFO SenderThread:6780 [sender.py:send_request_defer():391] handle sender defer: 3
64
+ 2021-11-08 08:08:31,851 INFO SenderThread:6780 [sender.py:transition_state():395] send defer: 4
65
+ 2021-11-08 08:08:31,851 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: defer
66
+ 2021-11-08 08:08:31,851 INFO HandlerThread:6780 [handler.py:handle_request_defer():147] handle defer: 4
67
+ 2021-11-08 08:08:31,851 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: defer
68
+ 2021-11-08 08:08:31,851 INFO SenderThread:6780 [sender.py:send_request_defer():391] handle sender defer: 4
69
+ 2021-11-08 08:08:31,900 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: poll_exit
70
+ 2021-11-08 08:08:32,052 INFO SenderThread:6780 [sender.py:transition_state():395] send defer: 5
71
+ 2021-11-08 08:08:32,053 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: poll_exit
72
+ 2021-11-08 08:08:32,053 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: defer
73
+ 2021-11-08 08:08:32,053 INFO HandlerThread:6780 [handler.py:handle_request_defer():147] handle defer: 5
74
+ 2021-11-08 08:08:32,053 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: defer
75
+ 2021-11-08 08:08:32,054 INFO SenderThread:6780 [sender.py:send_request_defer():391] handle sender defer: 5
76
+ 2021-11-08 08:08:32,054 INFO SenderThread:6780 [dir_watcher.py:finish():283] shutting down directory watcher
77
+ 2021-11-08 08:08:32,154 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: poll_exit
78
+ 2021-11-08 08:08:32,697 INFO Thread-8 :6780 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/files/wandb-summary.json
79
+ 2021-11-08 08:08:32,698 INFO SenderThread:6780 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/files/config.yaml
80
+ 2021-11-08 08:08:32,698 INFO SenderThread:6780 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/files/output.log
81
+ 2021-11-08 08:08:32,698 INFO SenderThread:6780 [dir_watcher.py:finish():313] scan: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/files
82
+ 2021-11-08 08:08:32,698 INFO SenderThread:6780 [dir_watcher.py:finish():327] scan save: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/files/requirements.txt requirements.txt
83
+ 2021-11-08 08:08:32,699 INFO SenderThread:6780 [dir_watcher.py:finish():327] scan save: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/files/output.log output.log
84
+ 2021-11-08 08:08:32,699 INFO SenderThread:6780 [dir_watcher.py:finish():327] scan save: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/files/wandb-metadata.json wandb-metadata.json
85
+ 2021-11-08 08:08:32,699 INFO SenderThread:6780 [dir_watcher.py:finish():327] scan save: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/files/config.yaml config.yaml
86
+ 2021-11-08 08:08:32,699 INFO SenderThread:6780 [dir_watcher.py:finish():327] scan save: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/files/wandb-summary.json wandb-summary.json
87
+ 2021-11-08 08:08:32,699 INFO SenderThread:6780 [sender.py:transition_state():395] send defer: 6
88
+ 2021-11-08 08:08:32,699 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: poll_exit
89
+ 2021-11-08 08:08:32,700 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: defer
90
+ 2021-11-08 08:08:32,700 INFO HandlerThread:6780 [handler.py:handle_request_defer():147] handle defer: 6
91
+ 2021-11-08 08:08:32,700 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: defer
92
+ 2021-11-08 08:08:32,700 INFO SenderThread:6780 [sender.py:send_request_defer():391] handle sender defer: 6
93
+ 2021-11-08 08:08:32,700 INFO SenderThread:6780 [file_pusher.py:finish():177] shutting down file pusher
94
+ 2021-11-08 08:08:32,804 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: poll_exit
95
+ 2021-11-08 08:08:32,805 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: poll_exit
96
+ 2021-11-08 08:08:32,908 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: poll_exit
97
+ 2021-11-08 08:08:32,908 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: poll_exit
98
+ 2021-11-08 08:08:33,009 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: poll_exit
99
+ 2021-11-08 08:08:33,010 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: poll_exit
100
+ 2021-11-08 08:08:33,111 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: poll_exit
101
+ 2021-11-08 08:08:33,111 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: poll_exit
102
+ 2021-11-08 08:08:33,204 INFO Thread-15 :6780 [upload_job.py:push():137] Uploaded file /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/files/wandb-summary.json
103
+ 2021-11-08 08:08:33,206 INFO Thread-12 :6780 [upload_job.py:push():137] Uploaded file /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/files/requirements.txt
104
+ 2021-11-08 08:08:33,213 INFO Thread-14 :6780 [upload_job.py:push():137] Uploaded file /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/files/config.yaml
105
+ 2021-11-08 08:08:33,213 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: poll_exit
106
+ 2021-11-08 08:08:33,213 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: poll_exit
107
+ 2021-11-08 08:08:33,263 INFO Thread-13 :6780 [upload_job.py:push():137] Uploaded file /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/files/output.log
108
+ 2021-11-08 08:08:33,315 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: poll_exit
109
+ 2021-11-08 08:08:33,315 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: poll_exit
110
+ 2021-11-08 08:08:33,416 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: poll_exit
111
+ 2021-11-08 08:08:33,417 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: poll_exit
112
+ 2021-11-08 08:08:33,463 INFO Thread-7 :6780 [sender.py:transition_state():395] send defer: 7
113
+ 2021-11-08 08:08:33,463 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: defer
114
+ 2021-11-08 08:08:33,464 INFO HandlerThread:6780 [handler.py:handle_request_defer():147] handle defer: 7
115
+ 2021-11-08 08:08:33,464 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: defer
116
+ 2021-11-08 08:08:33,464 INFO SenderThread:6780 [sender.py:send_request_defer():391] handle sender defer: 7
117
+ 2021-11-08 08:08:33,518 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: poll_exit
118
+ 2021-11-08 08:08:33,592 INFO SenderThread:6780 [sender.py:transition_state():395] send defer: 8
119
+ 2021-11-08 08:08:33,592 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: poll_exit
120
+ 2021-11-08 08:08:33,592 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: defer
121
+ 2021-11-08 08:08:33,592 INFO HandlerThread:6780 [handler.py:handle_request_defer():147] handle defer: 8
122
+ 2021-11-08 08:08:33,593 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: defer
123
+ 2021-11-08 08:08:33,593 INFO SenderThread:6780 [sender.py:send_request_defer():391] handle sender defer: 8
124
+ 2021-11-08 08:08:33,593 INFO SenderThread:6780 [sender.py:transition_state():395] send defer: 9
125
+ 2021-11-08 08:08:33,593 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: defer
126
+ 2021-11-08 08:08:33,593 INFO HandlerThread:6780 [handler.py:handle_request_defer():147] handle defer: 9
127
+ 2021-11-08 08:08:33,594 DEBUG SenderThread:6780 [sender.py:send():238] send: final
128
+ 2021-11-08 08:08:33,594 DEBUG SenderThread:6780 [sender.py:send():238] send: footer
129
+ 2021-11-08 08:08:33,594 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: defer
130
+ 2021-11-08 08:08:33,594 INFO SenderThread:6780 [sender.py:send_request_defer():391] handle sender defer: 9
131
+ 2021-11-08 08:08:33,693 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: poll_exit
132
+ 2021-11-08 08:08:33,693 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: poll_exit
133
+ 2021-11-08 08:08:33,694 INFO SenderThread:6780 [file_pusher.py:join():182] waiting for file pusher
134
+ 2021-11-08 08:08:33,969 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: get_summary
135
+ 2021-11-08 08:08:33,969 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: sampled_history
136
+ 2021-11-08 08:08:33,970 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: shutdown
137
+ 2021-11-08 08:08:33,970 INFO HandlerThread:6780 [handler.py:finish():731] shutting down handler
138
+ 2021-11-08 08:08:34,594 INFO WriterThread:6780 [datastore.py:close():281] close: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/run-3dz8poa1.wandb
139
+ 2021-11-08 08:08:34,968 INFO SenderThread:6780 [sender.py:finish():1091] shutting down sender
140
+ 2021-11-08 08:08:34,968 INFO SenderThread:6780 [file_pusher.py:finish():177] shutting down file pusher
141
+ 2021-11-08 08:08:34,968 INFO SenderThread:6780 [file_pusher.py:join():182] waiting for file pusher
142
+ 2021-11-08 08:08:34,971 INFO MainThread:6780 [internal.py:handle_exit():77] Internal process exited
wandb/run-20211108_080817-3dz8poa1/logs/debug.log ADDED
@@ -0,0 +1,129 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2021-11-08 08:08:17,854 INFO MainThread:5478 [wandb_setup.py:_flush():71] setting env: {}
2
+ 2021-11-08 08:08:17,855 INFO MainThread:5478 [wandb_setup.py:_flush():71] setting login settings: {}
3
+ 2021-11-08 08:08:17,855 INFO MainThread:5478 [wandb_init.py:_log_setup():357] Logging user logs to /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/logs/debug.log
4
+ 2021-11-08 08:08:17,855 INFO MainThread:5478 [wandb_init.py:_log_setup():358] Logging internal logs to /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/logs/debug-internal.log
5
+ 2021-11-08 08:08:17,855 INFO MainThread:5478 [wandb_init.py:init():390] calling init triggers
6
+ 2021-11-08 08:08:17,855 INFO MainThread:5478 [wandb_init.py:init():395] wandb.init called with sweep_config: {}
7
+ config: {}
8
+ 2021-11-08 08:08:17,855 INFO MainThread:5478 [wandb_init.py:init():435] starting backend
9
+ 2021-11-08 08:08:17,855 INFO MainThread:5478 [backend.py:_multiprocessing_setup():95] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
10
+ 2021-11-08 08:08:17,887 INFO MainThread:5478 [backend.py:ensure_launched():199] starting backend process...
11
+ 2021-11-08 08:08:17,917 INFO MainThread:5478 [backend.py:ensure_launched():204] started backend process with pid: 6780
12
+ 2021-11-08 08:08:17,919 INFO MainThread:5478 [wandb_init.py:init():444] backend started and connected
13
+ 2021-11-08 08:08:17,923 INFO MainThread:5478 [wandb_init.py:init():503] updated telemetry
14
+ 2021-11-08 08:08:18,001 INFO MainThread:5478 [wandb_init.py:init():533] communicating current version
15
+ 2021-11-08 08:08:18,527 INFO MainThread:5478 [wandb_init.py:init():538] got version response
16
+ 2021-11-08 08:08:18,527 INFO MainThread:5478 [wandb_init.py:init():548] communicating run to backend with 30 second timeout
17
+ 2021-11-08 08:08:18,688 INFO MainThread:5478 [wandb_init.py:init():576] starting run threads in backend
18
+ 2021-11-08 08:08:20,751 INFO MainThread:5478 [wandb_run.py:_console_start():1693] atexit reg
19
+ 2021-11-08 08:08:20,751 INFO MainThread:5478 [wandb_run.py:_redirect():1567] redirect: SettingsConsole.REDIRECT
20
+ 2021-11-08 08:08:20,752 INFO MainThread:5478 [wandb_run.py:_redirect():1572] Redirecting console.
21
+ 2021-11-08 08:08:20,754 INFO MainThread:5478 [wandb_run.py:_redirect():1628] Redirects installed.
22
+ 2021-11-08 08:08:20,754 INFO MainThread:5478 [wandb_init.py:init():603] run started, returning control to user process
23
+ 2021-11-08 08:08:20,760 INFO MainThread:5478 [wandb_run.py:_config_callback():911] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': True, 'do_eval': True, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 2, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 5e-05, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 3.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 0, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Nov08_08-08-10_t1v-n-ba7aa1a5-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 1000, 'logging_nan_inf_filter': True, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 25000, 'save_total_limit': 1, 'save_on_each_node': False, 'no_cuda': False, 'seed': 18, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'xpu_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 5000, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': 'en-ta-hi-te-from-100k', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': 'checkpoint_100000', 'hub_model_id': None, 'hub_strategy': 'HubStrategy.EVERY_SAVE', 'hub_token': None, 'gradient_checkpointing': False, 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': '', '_n_gpu': 0, '__cached__setup_devices': 'cpu'}
24
+ 2021-11-08 08:08:20,762 INFO MainThread:5478 [wandb_run.py:_config_callback():911] config_cb None None {'model_name_or_path': 'muril-bigbird', 'model_type': None, 'config_name': None, 'tokenizer_name': 'muril-bigbird', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'bfloat16'}
25
+ 2021-11-08 08:08:20,763 INFO MainThread:5478 [wandb_run.py:_config_callback():911] config_cb None None {'dataset_name': None, 'pretokenized': True, 'streaming': False, 'dataset_config_name': None, 'train_file': '/dev/shm/data-files', 'validation_file': '.', 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': None, 'mlm_probability': 0.15, 'pad_to_max_length': True, 'line_by_line': False}
26
+ 2021-11-08 08:08:29,658 INFO MainThread:5478 [wandb_run.py:_atexit_cleanup():1663] got exitcode: 255
27
+ 2021-11-08 08:08:29,658 INFO MainThread:5478 [wandb_run.py:_restore():1635] restore
28
+ 2021-11-08 08:08:31,799 INFO MainThread:5478 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
29
+ wandb_count: 1
30
+ }
31
+ pusher_stats {
32
+ uploaded_bytes: 1825
33
+ total_bytes: 1825
34
+ }
35
+
36
+ 2021-11-08 08:08:32,053 INFO MainThread:5478 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
37
+ wandb_count: 1
38
+ }
39
+ pusher_stats {
40
+ uploaded_bytes: 1825
41
+ total_bytes: 1825
42
+ }
43
+
44
+ 2021-11-08 08:08:32,703 INFO MainThread:5478 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
45
+ wandb_count: 5
46
+ }
47
+ pusher_stats {
48
+ uploaded_bytes: 1825
49
+ total_bytes: 18699
50
+ }
51
+
52
+ 2021-11-08 08:08:32,807 INFO MainThread:5478 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
53
+ wandb_count: 5
54
+ }
55
+ pusher_stats {
56
+ uploaded_bytes: 1825
57
+ total_bytes: 18699
58
+ }
59
+
60
+ 2021-11-08 08:08:32,909 INFO MainThread:5478 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
61
+ wandb_count: 5
62
+ }
63
+ pusher_stats {
64
+ uploaded_bytes: 18699
65
+ total_bytes: 18699
66
+ }
67
+
68
+ 2021-11-08 08:08:33,010 INFO MainThread:5478 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
69
+ wandb_count: 5
70
+ }
71
+ pusher_stats {
72
+ uploaded_bytes: 18699
73
+ total_bytes: 18699
74
+ }
75
+
76
+ 2021-11-08 08:08:33,112 INFO MainThread:5478 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
77
+ wandb_count: 5
78
+ }
79
+ pusher_stats {
80
+ uploaded_bytes: 18699
81
+ total_bytes: 18699
82
+ }
83
+
84
+ 2021-11-08 08:08:33,214 INFO MainThread:5478 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
85
+ wandb_count: 5
86
+ }
87
+ pusher_stats {
88
+ uploaded_bytes: 18699
89
+ total_bytes: 18699
90
+ }
91
+
92
+ 2021-11-08 08:08:33,315 INFO MainThread:5478 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
93
+ wandb_count: 5
94
+ }
95
+ pusher_stats {
96
+ uploaded_bytes: 18699
97
+ total_bytes: 18699
98
+ }
99
+
100
+ 2021-11-08 08:08:33,417 INFO MainThread:5478 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
101
+ wandb_count: 5
102
+ }
103
+ pusher_stats {
104
+ uploaded_bytes: 18699
105
+ total_bytes: 18699
106
+ }
107
+
108
+ 2021-11-08 08:08:33,592 INFO MainThread:5478 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
109
+ wandb_count: 5
110
+ }
111
+ pusher_stats {
112
+ uploaded_bytes: 18699
113
+ total_bytes: 18699
114
+ }
115
+
116
+ 2021-11-08 08:08:33,968 INFO MainThread:5478 [wandb_run.py:_wait_for_finish():1793] got exit ret: done: true
117
+ exit_result {
118
+ }
119
+ file_counts {
120
+ wandb_count: 5
121
+ }
122
+ pusher_stats {
123
+ uploaded_bytes: 18699
124
+ total_bytes: 18699
125
+ }
126
+ local_info {
127
+ }
128
+
129
+ 2021-11-08 08:08:35,338 INFO MainThread:5478 [wandb_run.py:_append_files():2061] logging synced files
wandb/run-20211108_080817-3dz8poa1/run-3dz8poa1.wandb ADDED
Binary file (13.5 kB). View file
 
wandb/run-20211108_080940-r077t6tw/files/config.yaml ADDED
@@ -0,0 +1,347 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ wandb_version: 1
2
+
3
+ __cached__setup_devices:
4
+ desc: null
5
+ value: cpu
6
+ _n_gpu:
7
+ desc: null
8
+ value: 0
9
+ _wandb:
10
+ desc: null
11
+ value:
12
+ cli_version: 0.12.6
13
+ framework: huggingface
14
+ huggingface_version: 4.13.0.dev0
15
+ is_jupyter_run: false
16
+ is_kaggle_kernel: true
17
+ python_version: 3.8.10
18
+ start_time: 1636358980
19
+ t:
20
+ 1:
21
+ - 1
22
+ - 2
23
+ - 3
24
+ - 11
25
+ - 12
26
+ 2:
27
+ - 1
28
+ - 2
29
+ - 3
30
+ - 11
31
+ - 12
32
+ 3:
33
+ - 13
34
+ 4: 3.8.10
35
+ 5: 0.12.6
36
+ 6: 4.13.0.dev0
37
+ 8:
38
+ - 2
39
+ - 5
40
+ adafactor:
41
+ desc: null
42
+ value: false
43
+ adam_beta1:
44
+ desc: null
45
+ value: 0.9
46
+ adam_beta2:
47
+ desc: null
48
+ value: 0.999
49
+ adam_epsilon:
50
+ desc: null
51
+ value: 1.0e-08
52
+ cache_dir:
53
+ desc: null
54
+ value: null
55
+ config_name:
56
+ desc: null
57
+ value: null
58
+ dataloader_drop_last:
59
+ desc: null
60
+ value: false
61
+ dataloader_num_workers:
62
+ desc: null
63
+ value: 0
64
+ dataloader_pin_memory:
65
+ desc: null
66
+ value: true
67
+ dataset_config_name:
68
+ desc: null
69
+ value: null
70
+ dataset_name:
71
+ desc: null
72
+ value: null
73
+ ddp_find_unused_parameters:
74
+ desc: null
75
+ value: null
76
+ debug:
77
+ desc: null
78
+ value: []
79
+ deepspeed:
80
+ desc: null
81
+ value: null
82
+ disable_tqdm:
83
+ desc: null
84
+ value: false
85
+ do_eval:
86
+ desc: null
87
+ value: true
88
+ do_predict:
89
+ desc: null
90
+ value: false
91
+ do_train:
92
+ desc: null
93
+ value: true
94
+ dtype:
95
+ desc: null
96
+ value: bfloat16
97
+ eval_accumulation_steps:
98
+ desc: null
99
+ value: null
100
+ eval_steps:
101
+ desc: null
102
+ value: 5000
103
+ evaluation_strategy:
104
+ desc: null
105
+ value: IntervalStrategy.NO
106
+ fp16:
107
+ desc: null
108
+ value: false
109
+ fp16_backend:
110
+ desc: null
111
+ value: auto
112
+ fp16_full_eval:
113
+ desc: null
114
+ value: false
115
+ fp16_opt_level:
116
+ desc: null
117
+ value: O1
118
+ gradient_accumulation_steps:
119
+ desc: null
120
+ value: 1
121
+ gradient_checkpointing:
122
+ desc: null
123
+ value: false
124
+ greater_is_better:
125
+ desc: null
126
+ value: null
127
+ group_by_length:
128
+ desc: null
129
+ value: false
130
+ hub_model_id:
131
+ desc: null
132
+ value: null
133
+ hub_strategy:
134
+ desc: null
135
+ value: HubStrategy.EVERY_SAVE
136
+ hub_token:
137
+ desc: null
138
+ value: null
139
+ ignore_data_skip:
140
+ desc: null
141
+ value: false
142
+ label_names:
143
+ desc: null
144
+ value: null
145
+ label_smoothing_factor:
146
+ desc: null
147
+ value: 0.0
148
+ learning_rate:
149
+ desc: null
150
+ value: 5.0e-05
151
+ length_column_name:
152
+ desc: null
153
+ value: length
154
+ line_by_line:
155
+ desc: null
156
+ value: false
157
+ load_best_model_at_end:
158
+ desc: null
159
+ value: false
160
+ local_rank:
161
+ desc: null
162
+ value: -1
163
+ log_level:
164
+ desc: null
165
+ value: -1
166
+ log_level_replica:
167
+ desc: null
168
+ value: -1
169
+ log_on_each_node:
170
+ desc: null
171
+ value: true
172
+ logging_dir:
173
+ desc: null
174
+ value: ./runs/Nov08_08-09-31_t1v-n-ba7aa1a5-w-0
175
+ logging_first_step:
176
+ desc: null
177
+ value: false
178
+ logging_nan_inf_filter:
179
+ desc: null
180
+ value: true
181
+ logging_steps:
182
+ desc: null
183
+ value: 1000
184
+ logging_strategy:
185
+ desc: null
186
+ value: IntervalStrategy.STEPS
187
+ lr_scheduler_type:
188
+ desc: null
189
+ value: SchedulerType.LINEAR
190
+ max_grad_norm:
191
+ desc: null
192
+ value: 1.0
193
+ max_seq_length:
194
+ desc: null
195
+ value: 4096
196
+ max_steps:
197
+ desc: null
198
+ value: -1
199
+ metric_for_best_model:
200
+ desc: null
201
+ value: null
202
+ mlm_probability:
203
+ desc: null
204
+ value: 0.15
205
+ model_name_or_path:
206
+ desc: null
207
+ value: muril-bigbird
208
+ model_type:
209
+ desc: null
210
+ value: null
211
+ mp_parameters:
212
+ desc: null
213
+ value: ''
214
+ no_cuda:
215
+ desc: null
216
+ value: false
217
+ num_train_epochs:
218
+ desc: null
219
+ value: 3.0
220
+ output_dir:
221
+ desc: null
222
+ value: ./
223
+ overwrite_cache:
224
+ desc: null
225
+ value: false
226
+ overwrite_output_dir:
227
+ desc: null
228
+ value: true
229
+ pad_to_max_length:
230
+ desc: null
231
+ value: true
232
+ past_index:
233
+ desc: null
234
+ value: -1
235
+ per_device_eval_batch_size:
236
+ desc: null
237
+ value: 2
238
+ per_device_train_batch_size:
239
+ desc: null
240
+ value: 1
241
+ per_gpu_eval_batch_size:
242
+ desc: null
243
+ value: null
244
+ per_gpu_train_batch_size:
245
+ desc: null
246
+ value: null
247
+ prediction_loss_only:
248
+ desc: null
249
+ value: false
250
+ preprocessing_num_workers:
251
+ desc: null
252
+ value: null
253
+ pretokenized:
254
+ desc: null
255
+ value: true
256
+ push_to_hub:
257
+ desc: null
258
+ value: true
259
+ push_to_hub_model_id:
260
+ desc: null
261
+ value: null
262
+ push_to_hub_organization:
263
+ desc: null
264
+ value: null
265
+ push_to_hub_token:
266
+ desc: null
267
+ value: null
268
+ remove_unused_columns:
269
+ desc: null
270
+ value: true
271
+ report_to:
272
+ desc: null
273
+ value:
274
+ - tensorboard
275
+ - wandb
276
+ resume_from_checkpoint:
277
+ desc: null
278
+ value: checkpoint_100000
279
+ run_name:
280
+ desc: null
281
+ value: en-ta-hi-te-from-100k
282
+ save_on_each_node:
283
+ desc: null
284
+ value: false
285
+ save_steps:
286
+ desc: null
287
+ value: 25000
288
+ save_strategy:
289
+ desc: null
290
+ value: IntervalStrategy.STEPS
291
+ save_total_limit:
292
+ desc: null
293
+ value: 1
294
+ seed:
295
+ desc: null
296
+ value: 18
297
+ sharded_ddp:
298
+ desc: null
299
+ value: []
300
+ skip_memory_metrics:
301
+ desc: null
302
+ value: true
303
+ streaming:
304
+ desc: null
305
+ value: false
306
+ tokenizer_name:
307
+ desc: null
308
+ value: muril-bigbird
309
+ tpu_metrics_debug:
310
+ desc: null
311
+ value: false
312
+ tpu_num_cores:
313
+ desc: null
314
+ value: null
315
+ train_file:
316
+ desc: null
317
+ value: /dev/shm/data-files
318
+ train_ref_file:
319
+ desc: null
320
+ value: null
321
+ use_fast_tokenizer:
322
+ desc: null
323
+ value: true
324
+ use_legacy_prediction_loop:
325
+ desc: null
326
+ value: false
327
+ validation_file:
328
+ desc: null
329
+ value: .
330
+ validation_ref_file:
331
+ desc: null
332
+ value: null
333
+ validation_split_percentage:
334
+ desc: null
335
+ value: 5
336
+ warmup_ratio:
337
+ desc: null
338
+ value: 0.0
339
+ warmup_steps:
340
+ desc: null
341
+ value: 0
342
+ weight_decay:
343
+ desc: null
344
+ value: 0.01
345
+ xpu_backend:
346
+ desc: null
347
+ value: null
wandb/run-20211108_080940-r077t6tw/files/output.log ADDED
@@ -0,0 +1,108 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Traceback (most recent call last):
2
+ File "run_muril_bb_flax_mlm.py", line 548, in <module>
3
+ model = FlaxAutoModelForMaskedLM.from_pretrained(
4
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/transformers/models/auto/auto_factory.py", line 419, in from_pretrained
5
+ return model_class.from_pretrained(pretrained_model_name_or_path, *model_args, config=config, **kwargs)
6
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/transformers/modeling_flax_utils.py", line 351, in from_pretrained
7
+ model = cls(config, *model_args, **model_kwargs)
8
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/transformers/models/big_bird/modeling_flax_big_bird.py", line 1359, in __init__
9
+ super().__init__(config, module, input_shape=input_shape, seed=seed, dtype=dtype)
10
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/transformers/modeling_flax_utils.py", line 106, in __init__
11
+ random_params = self.init_weights(self.key, input_shape)
12
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/transformers/models/big_bird/modeling_flax_big_bird.py", line 1371, in init_weights
13
+ return self.module.init(rngs, input_ids, attention_mask, token_type_ids, position_ids, return_dict=False)[
14
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/_src/traceback_util.py", line 162, in reraise_with_filtered_traceback
15
+ return fun(*args, **kwargs)
16
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 1122, in init
17
+ _, v_out = self.init_with_output(
18
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/_src/traceback_util.py", line 162, in reraise_with_filtered_traceback
19
+ return fun(*args, **kwargs)
20
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 1091, in init_with_output
21
+ return self.apply(
22
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/_src/traceback_util.py", line 162, in reraise_with_filtered_traceback
23
+ return fun(*args, **kwargs)
24
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 1058, in apply
25
+ return apply(
26
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/core/scope.py", line 706, in wrapper
27
+ y = fn(root, *args, **kwargs)
28
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 1313, in scope_fn
29
+ return fn(module.clone(parent=scope), *args, **kwargs)
30
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/transforms.py", line 883, in wrapped_fn
31
+ return prewrapped_fn(self, *args, **kwargs)
32
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 318, in wrapped_module_method
33
+ return self._call_wrapped_method(fun, args, kwargs)
34
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 603, in _call_wrapped_method
35
+ y = fun(self, *args, **kwargs)
36
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/transformers/models/big_bird/modeling_flax_big_bird.py", line 1607, in __call__
37
+ outputs = self.bert(
38
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/transforms.py", line 883, in wrapped_fn
39
+ return prewrapped_fn(self, *args, **kwargs)
40
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 318, in wrapped_module_method
41
+ return self._call_wrapped_method(fun, args, kwargs)
42
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 603, in _call_wrapped_method
43
+ y = fun(self, *args, **kwargs)
44
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/transformers/models/big_bird/modeling_flax_big_bird.py", line 1449, in __call__
45
+ hidden_states = self.embeddings(
46
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/transforms.py", line 883, in wrapped_fn
47
+ return prewrapped_fn(self, *args, **kwargs)
48
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 318, in wrapped_module_method
49
+ return self._call_wrapped_method(fun, args, kwargs)
50
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 603, in _call_wrapped_method
51
+ y = fun(self, *args, **kwargs)
52
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/transformers/models/big_bird/modeling_flax_big_bird.py", line 208, in __call__
53
+ token_type_embeddings = self.token_type_embeddings(token_type_ids.astype("i4"))
54
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/transforms.py", line 883, in wrapped_fn
55
+ return prewrapped_fn(self, *args, **kwargs)
56
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 318, in wrapped_module_method
57
+ return self._call_wrapped_method(fun, args, kwargs)
58
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 594, in _call_wrapped_method
59
+ self._try_setup()
60
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 790, in _try_setup
61
+ self.setup()
62
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 318, in wrapped_module_method
63
+ return self._call_wrapped_method(fun, args, kwargs)
64
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 603, in _call_wrapped_method
65
+ y = fun(self, *args, **kwargs)
66
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/linear.py", line 405, in setup
67
+ self.embedding = self.param('embedding',
68
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 896, in param
69
+ v = self.scope.param(name, init_fn, *init_args)
70
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/core/scope.py", line 643, in param
71
+ value = init_fn(self.make_rng('params'), *init_args)
72
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/_src/nn/initializers.py", line 44, in init
73
+ return random.normal(key, shape, dtype) * stddev
74
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/_src/random.py", line 521, in normal
75
+ return _normal(key, shape, dtype) # type: ignore
76
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/_src/traceback_util.py", line 162, in reraise_with_filtered_traceback
77
+ return fun(*args, **kwargs)
78
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/_src/api.py", line 416, in cache_miss
79
+ out_flat = xla.xla_call(
80
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/core.py", line 1632, in bind
81
+ return call_bind(self, fun, *args, **params)
82
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/core.py", line 1623, in call_bind
83
+ outs = primitive.process(top_trace, fun, tracers, params)
84
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/core.py", line 1635, in process
85
+ return trace.process_call(self, fun, tracers, params)
86
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/core.py", line 627, in process_call
87
+ return primitive.impl(f, *tracers, **params)
88
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/interpreters/xla.py", line 581, in _xla_call_impl
89
+ compiled_fun = _xla_callable(fun, device, backend, name, donated_invars,
90
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/linear_util.py", line 263, in memoized_fun
91
+ ans = call(fun, *args)
92
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/interpreters/xla.py", line 653, in _xla_callable_uncached
93
+ return lower_xla_callable(fun, device, backend, name, donated_invars,
94
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/interpreters/xla.py", line 723, in lower_xla_callable
95
+ out_nodes = jaxpr_subcomp(ctx, jaxpr, xla_consts, *xla_args)
96
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/interpreters/xla.py", line 471, in jaxpr_subcomp
97
+ ans = rule(ctx, map(aval, eqn.invars), map(aval, eqn.outvars),
98
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/interpreters/xla.py", line 1217, in f_new
99
+ return jaxpr_subcomp(ctx, jaxpr, _xla_consts(ctx.builder, consts),
100
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/interpreters/xla.py", line 456, in jaxpr_subcomp
101
+ op_metadata = make_op_metadata(
102
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/interpreters/xla.py", line 122, in make_op_metadata
103
+ frame = source_info_util.user_frame(source_info) if source_info else None
104
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/_src/source_info_util.py", line 48, in user_frame
105
+ return next(user_frames(source_info), None)
106
+ File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/_src/source_info_util.py", line 44, in user_frames
107
+ return (x for x in (source_info.frames if source_info else [])
108
+ KeyboardInterrupt