Hugging Face's logo

sptrodon
/
Optimum-Graphcore-Demo

Question Answering
Transformers
PyTorch
Graphcore
bert
Model card Files
xet
Community
1
sptrodon commited on
Commit
b4fefc8
路
1 Parent(s): 096a96e

Upload with huggingface_hub

Browse files
Files changed (20) hide show
  1. checkpoint-345/config.json +26 -0
  2. checkpoint-345/ipu_config.json +33 -0
  3. checkpoint-345/optimizer.pt +3 -0
  4. checkpoint-345/pytorch_model.bin +3 -0
  5. checkpoint-345/rng_state.pth +3 -0
  6. checkpoint-345/scheduler.pt +3 -0
  7. checkpoint-345/trainer_state.json +95 -0
  8. checkpoint-345/training_args.bin +3 -0
  9. checkpoint-690/config.json +26 -0
  10. checkpoint-690/ipu_config.json +33 -0
  11. checkpoint-690/optimizer.pt +3 -0
  12. checkpoint-690/pytorch_model.bin +3 -0
  13. checkpoint-690/rng_state.pth +3 -0
  14. checkpoint-690/scheduler.pt +3 -0
  15. checkpoint-690/trainer_state.json +179 -0
  16. checkpoint-690/training_args.bin +3 -0
  17. config.json +26 -0
  18. ipu_config.json +33 -0
  19. pytorch_model.bin +3 -0
  20. training_args.bin +3 -0
checkpoint-345/config.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "Graphcore/bert-large-uncased",
3
+ "architectures": [
4
+ "PoptorchPipelinedBertForQuestionAnswering"
5
+ ],
6
+ "attention_probs_dropout_prob": 0.1,
7
+ "classifier_dropout": null,
8
+ "gradient_checkpointing": false,
9
+ "hidden_act": "gelu",
10
+ "hidden_dropout_prob": 0.1,
11
+ "hidden_size": 1024,
12
+ "initializer_range": 0.02,
13
+ "intermediate_size": 4096,
14
+ "layer_norm_eps": 0.001,
15
+ "max_position_embeddings": 512,
16
+ "model_type": "bert",
17
+ "num_attention_heads": 16,
18
+ "num_hidden_layers": 24,
19
+ "pad_token_id": 0,
20
+ "position_embedding_type": "absolute",
21
+ "torch_dtype": "float16",
22
+ "transformers_version": "4.20.1",
23
+ "type_vocab_size": 2,
24
+ "use_cache": true,
25
+ "vocab_size": 30522
26
+ }
checkpoint-345/ipu_config.json ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "auto_loss_scaling": false,
3
+ "device_iterations": 1,
4
+ "embedding_serialization_factor": 2,
5
+ "enable_half_partials": true,
6
+ "executable_cache_dir": "/tmp/exe_cache/introduction_to_optimum_graphcore",
7
+ "execute_encoder_on_cpu_for_generation": false,
8
+ "gradient_accumulation_steps": 256,
9
+ "inference_device_iterations": 5,
10
+ "inference_replication_factor": 1,
11
+ "ipus_per_replica": 4,
12
+ "layers_per_ipu": [
13
+ 3,
14
+ 7,
15
+ 7,
16
+ 7
17
+ ],
18
+ "matmul_proportion": [
19
+ 0.15,
20
+ 0.18,
21
+ 0.2,
22
+ 0.25
23
+ ],
24
+ "optimizer_state_offchip": true,
25
+ "optimum_version": "1.6.1",
26
+ "output_mode": "final",
27
+ "recompute_checkpoint_every_layer": true,
28
+ "replicated_tensor_sharding": false,
29
+ "replication_factor": 1,
30
+ "seed": 42,
31
+ "sharded_execution_for_inference": false,
32
+ "transformers_version": "4.20.1"
33
+ }
checkpoint-345/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9cab570fc93cd09232d801f1b76de79820f31a1f2f9b9c56ff67b12a73ab97d6
3
+ size 2005478613
checkpoint-345/pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:18ae1bb25a3706c33ee88b1e1641f43752ad7b67f268d65a3d7e230418d85474
3
+ size 668321389
checkpoint-345/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ee7ffc5810caeabbdfa3adb23763a98c55c2da84f3150a7bc3a09b3390dd145a
3
+ size 13553
checkpoint-345/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e4b7ebbcaf5df410427daf0fe4e6c4360e29db9a3a70e622d1a6d9edfd24355a
3
+ size 627
checkpoint-345/trainer_state.json ADDED
@@ -0,0 +1,95 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 1.0,
5
+ "global_step": 345,
6
+ "is_hyper_param_search": false,
7
+ "is_local_process_zero": true,
8
+ "is_world_process_zero": true,
9
+ "log_history": [
10
+ {
11
+ "epoch": 0.07,
12
+ "learning_rate": 0.0001927536231884058,
13
+ "loss": 2.497,
14
+ "step": 25
15
+ },
16
+ {
17
+ "epoch": 0.14,
18
+ "learning_rate": 0.0001855072463768116,
19
+ "loss": 1.0576,
20
+ "step": 50
21
+ },
22
+ {
23
+ "epoch": 0.22,
24
+ "learning_rate": 0.0001782608695652174,
25
+ "loss": 1.063,
26
+ "step": 75
27
+ },
28
+ {
29
+ "epoch": 0.29,
30
+ "learning_rate": 0.0001710144927536232,
31
+ "loss": 1.5434,
32
+ "step": 100
33
+ },
34
+ {
35
+ "epoch": 0.36,
36
+ "learning_rate": 0.000163768115942029,
37
+ "loss": 1.1751,
38
+ "step": 125
39
+ },
40
+ {
41
+ "epoch": 0.43,
42
+ "learning_rate": 0.0001565217391304348,
43
+ "loss": 0.9868,
44
+ "step": 150
45
+ },
46
+ {
47
+ "epoch": 0.51,
48
+ "learning_rate": 0.00014927536231884058,
49
+ "loss": 1.0466,
50
+ "step": 175
51
+ },
52
+ {
53
+ "epoch": 0.58,
54
+ "learning_rate": 0.00014202898550724638,
55
+ "loss": 0.7962,
56
+ "step": 200
57
+ },
58
+ {
59
+ "epoch": 0.65,
60
+ "learning_rate": 0.0001347826086956522,
61
+ "loss": 1.0158,
62
+ "step": 225
63
+ },
64
+ {
65
+ "epoch": 0.72,
66
+ "learning_rate": 0.00012753623188405797,
67
+ "loss": 0.8366,
68
+ "step": 250
69
+ },
70
+ {
71
+ "epoch": 0.8,
72
+ "learning_rate": 0.00012028985507246378,
73
+ "loss": 1.2794,
74
+ "step": 275
75
+ },
76
+ {
77
+ "epoch": 0.87,
78
+ "learning_rate": 0.00011304347826086956,
79
+ "loss": 1.6535,
80
+ "step": 300
81
+ },
82
+ {
83
+ "epoch": 0.94,
84
+ "learning_rate": 0.00010579710144927538,
85
+ "loss": 0.878,
86
+ "step": 325
87
+ }
88
+ ],
89
+ "max_steps": 690,
90
+ "num_train_epochs": 2,
91
+ "start_time": 1678594586.128044,
92
+ "total_flos": 1.5748493096591032e+19,
93
+ "trial_name": null,
94
+ "trial_params": null
95
+ }
checkpoint-345/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3081639520de5c1f5c8a12096f2aefdcc1fbcd041daa5e0a52782bdd2823f1ae
3
+ size 2747
checkpoint-690/config.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "Graphcore/bert-large-uncased",
3
+ "architectures": [
4
+ "PoptorchPipelinedBertForQuestionAnswering"
5
+ ],
6
+ "attention_probs_dropout_prob": 0.1,
7
+ "classifier_dropout": null,
8
+ "gradient_checkpointing": false,
9
+ "hidden_act": "gelu",
10
+ "hidden_dropout_prob": 0.1,
11
+ "hidden_size": 1024,
12
+ "initializer_range": 0.02,
13
+ "intermediate_size": 4096,
14
+ "layer_norm_eps": 0.001,
15
+ "max_position_embeddings": 512,
16
+ "model_type": "bert",
17
+ "num_attention_heads": 16,
18
+ "num_hidden_layers": 24,
19
+ "pad_token_id": 0,
20
+ "position_embedding_type": "absolute",
21
+ "torch_dtype": "float16",
22
+ "transformers_version": "4.20.1",
23
+ "type_vocab_size": 2,
24
+ "use_cache": true,
25
+ "vocab_size": 30522
26
+ }
checkpoint-690/ipu_config.json ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "auto_loss_scaling": false,
3
+ "device_iterations": 1,
4
+ "embedding_serialization_factor": 2,
5
+ "enable_half_partials": true,
6
+ "executable_cache_dir": "/tmp/exe_cache/introduction_to_optimum_graphcore",
7
+ "execute_encoder_on_cpu_for_generation": false,
8
+ "gradient_accumulation_steps": 256,
9
+ "inference_device_iterations": 5,
10
+ "inference_replication_factor": 1,
11
+ "ipus_per_replica": 4,
12
+ "layers_per_ipu": [
13
+ 3,
14
+ 7,
15
+ 7,
16
+ 7
17
+ ],
18
+ "matmul_proportion": [
19
+ 0.15,
20
+ 0.18,
21
+ 0.2,
22
+ 0.25
23
+ ],
24
+ "optimizer_state_offchip": true,
25
+ "optimum_version": "1.6.1",
26
+ "output_mode": "final",
27
+ "recompute_checkpoint_every_layer": true,
28
+ "replicated_tensor_sharding": false,
29
+ "replication_factor": 1,
30
+ "seed": 42,
31
+ "sharded_execution_for_inference": false,
32
+ "transformers_version": "4.20.1"
33
+ }
checkpoint-690/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a7359d64cf80b9a580a0d983fafbdb051917346193468994ae564d989b69d655
3
+ size 2005478613
checkpoint-690/pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5f384c444fd59609a964def97aa72c65f137875bd481e99af953082f996446c4
3
+ size 668321389
checkpoint-690/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7f362665c85f7d92ec1a357fc4e578a4577ab5e1c41aa343b3753ce3487396f4
3
+ size 13553
checkpoint-690/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b7f9e754df881686260f74a9c79c598c26991e8db0261f411577cf8ccd137b8f
3
+ size 627
checkpoint-690/trainer_state.json ADDED
@@ -0,0 +1,179 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 2.0,
5
+ "global_step": 690,
6
+ "is_hyper_param_search": false,
7
+ "is_local_process_zero": true,
8
+ "is_world_process_zero": true,
9
+ "log_history": [
10
+ {
11
+ "epoch": 0.07,
12
+ "learning_rate": 0.0001927536231884058,
13
+ "loss": 2.497,
14
+ "step": 25
15
+ },
16
+ {
17
+ "epoch": 0.14,
18
+ "learning_rate": 0.0001855072463768116,
19
+ "loss": 1.0576,
20
+ "step": 50
21
+ },
22
+ {
23
+ "epoch": 0.22,
24
+ "learning_rate": 0.0001782608695652174,
25
+ "loss": 1.063,
26
+ "step": 75
27
+ },
28
+ {
29
+ "epoch": 0.29,
30
+ "learning_rate": 0.0001710144927536232,
31
+ "loss": 1.5434,
32
+ "step": 100
33
+ },
34
+ {
35
+ "epoch": 0.36,
36
+ "learning_rate": 0.000163768115942029,
37
+ "loss": 1.1751,
38
+ "step": 125
39
+ },
40
+ {
41
+ "epoch": 0.43,
42
+ "learning_rate": 0.0001565217391304348,
43
+ "loss": 0.9868,
44
+ "step": 150
45
+ },
46
+ {
47
+ "epoch": 0.51,
48
+ "learning_rate": 0.00014927536231884058,
49
+ "loss": 1.0466,
50
+ "step": 175
51
+ },
52
+ {
53
+ "epoch": 0.58,
54
+ "learning_rate": 0.00014202898550724638,
55
+ "loss": 0.7962,
56
+ "step": 200
57
+ },
58
+ {
59
+ "epoch": 0.65,
60
+ "learning_rate": 0.0001347826086956522,
61
+ "loss": 1.0158,
62
+ "step": 225
63
+ },
64
+ {
65
+ "epoch": 0.72,
66
+ "learning_rate": 0.00012753623188405797,
67
+ "loss": 0.8366,
68
+ "step": 250
69
+ },
70
+ {
71
+ "epoch": 0.8,
72
+ "learning_rate": 0.00012028985507246378,
73
+ "loss": 1.2794,
74
+ "step": 275
75
+ },
76
+ {
77
+ "epoch": 0.87,
78
+ "learning_rate": 0.00011304347826086956,
79
+ "loss": 1.6535,
80
+ "step": 300
81
+ },
82
+ {
83
+ "epoch": 0.94,
84
+ "learning_rate": 0.00010579710144927538,
85
+ "loss": 0.878,
86
+ "step": 325
87
+ },
88
+ {
89
+ "epoch": 1.01,
90
+ "learning_rate": 9.855072463768117e-05,
91
+ "loss": 0.6866,
92
+ "step": 350
93
+ },
94
+ {
95
+ "epoch": 1.09,
96
+ "learning_rate": 9.130434782608696e-05,
97
+ "loss": 0.5021,
98
+ "step": 375
99
+ },
100
+ {
101
+ "epoch": 1.16,
102
+ "learning_rate": 8.405797101449276e-05,
103
+ "loss": 0.4383,
104
+ "step": 400
105
+ },
106
+ {
107
+ "epoch": 1.23,
108
+ "learning_rate": 7.681159420289855e-05,
109
+ "loss": 0.4791,
110
+ "step": 425
111
+ },
112
+ {
113
+ "epoch": 1.3,
114
+ "learning_rate": 6.956521739130436e-05,
115
+ "loss": 0.5432,
116
+ "step": 450
117
+ },
118
+ {
119
+ "epoch": 1.38,
120
+ "learning_rate": 6.231884057971015e-05,
121
+ "loss": 0.3643,
122
+ "step": 475
123
+ },
124
+ {
125
+ "epoch": 1.45,
126
+ "learning_rate": 5.507246376811594e-05,
127
+ "loss": 1.4377,
128
+ "step": 500
129
+ },
130
+ {
131
+ "epoch": 1.52,
132
+ "learning_rate": 4.782608695652174e-05,
133
+ "loss": 0.4916,
134
+ "step": 525
135
+ },
136
+ {
137
+ "epoch": 1.59,
138
+ "learning_rate": 4.057971014492754e-05,
139
+ "loss": 0.5831,
140
+ "step": 550
141
+ },
142
+ {
143
+ "epoch": 1.67,
144
+ "learning_rate": 3.3333333333333335e-05,
145
+ "loss": 0.7232,
146
+ "step": 575
147
+ },
148
+ {
149
+ "epoch": 1.74,
150
+ "learning_rate": 2.608695652173913e-05,
151
+ "loss": 0.7072,
152
+ "step": 600
153
+ },
154
+ {
155
+ "epoch": 1.81,
156
+ "learning_rate": 1.8840579710144928e-05,
157
+ "loss": 0.5308,
158
+ "step": 625
159
+ },
160
+ {
161
+ "epoch": 1.88,
162
+ "learning_rate": 1.1594202898550725e-05,
163
+ "loss": 0.4856,
164
+ "step": 650
165
+ },
166
+ {
167
+ "epoch": 1.96,
168
+ "learning_rate": 4.347826086956522e-06,
169
+ "loss": 0.8379,
170
+ "step": 675
171
+ }
172
+ ],
173
+ "max_steps": 690,
174
+ "num_train_epochs": 2,
175
+ "start_time": 1678594586.128044,
176
+ "total_flos": 3.1496986193182065e+19,
177
+ "trial_name": null,
178
+ "trial_params": null
179
+ }
checkpoint-690/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3081639520de5c1f5c8a12096f2aefdcc1fbcd041daa5e0a52782bdd2823f1ae
3
+ size 2747
config.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "Graphcore/bert-large-uncased",
3
+ "architectures": [
4
+ "PoptorchPipelinedBertForQuestionAnswering"
5
+ ],
6
+ "attention_probs_dropout_prob": 0.1,
7
+ "classifier_dropout": null,
8
+ "gradient_checkpointing": false,
9
+ "hidden_act": "gelu",
10
+ "hidden_dropout_prob": 0.1,
11
+ "hidden_size": 1024,
12
+ "initializer_range": 0.02,
13
+ "intermediate_size": 4096,
14
+ "layer_norm_eps": 0.001,
15
+ "max_position_embeddings": 512,
16
+ "model_type": "bert",
17
+ "num_attention_heads": 16,
18
+ "num_hidden_layers": 24,
19
+ "pad_token_id": 0,
20
+ "position_embedding_type": "absolute",
21
+ "torch_dtype": "float16",
22
+ "transformers_version": "4.20.1",
23
+ "type_vocab_size": 2,
24
+ "use_cache": true,
25
+ "vocab_size": 30522
26
+ }
ipu_config.json ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "auto_loss_scaling": false,
3
+ "device_iterations": 1,
4
+ "embedding_serialization_factor": 2,
5
+ "enable_half_partials": true,
6
+ "executable_cache_dir": "/tmp/exe_cache/introduction_to_optimum_graphcore",
7
+ "execute_encoder_on_cpu_for_generation": false,
8
+ "gradient_accumulation_steps": 256,
9
+ "inference_device_iterations": 5,
10
+ "inference_replication_factor": 1,
11
+ "ipus_per_replica": 4,
12
+ "layers_per_ipu": [
13
+ 3,
14
+ 7,
15
+ 7,
16
+ 7
17
+ ],
18
+ "matmul_proportion": [
19
+ 0.15,
20
+ 0.18,
21
+ 0.2,
22
+ 0.25
23
+ ],
24
+ "optimizer_state_offchip": true,
25
+ "optimum_version": "1.6.1",
26
+ "output_mode": "final",
27
+ "recompute_checkpoint_every_layer": true,
28
+ "replicated_tensor_sharding": false,
29
+ "replication_factor": 1,
30
+ "seed": 42,
31
+ "sharded_execution_for_inference": false,
32
+ "transformers_version": "4.20.1"
33
+ }
pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5f384c444fd59609a964def97aa72c65f137875bd481e99af953082f996446c4
3
+ size 668321389
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3081639520de5c1f5c8a12096f2aefdcc1fbcd041daa5e0a52782bdd2823f1ae
3
+ size 2747