Hugging Face's logo

mekaneeky
/
dataset-tags-expanded-tokenizer-oversampling

Model card Files
xet
Community
azawahry commited on
Commit
fa31ffd
·
1 Parent(s): e77268d

Initial commit

Browse files
Files changed (30) hide show
  1. checkpoint-100/added_tokens.json +44 -0
  2. checkpoint-100/config.json +61 -0
  3. checkpoint-100/generation_config.json +15 -0
  4. checkpoint-100/optimizer.pt +3 -0
  5. checkpoint-100/pytorch_model.bin +3 -0
  6. checkpoint-100/rng_state.pth +3 -0
  7. checkpoint-100/scaler.pt +3 -0
  8. checkpoint-100/scheduler.pt +3 -0
  9. checkpoint-100/source.spm +0 -0
  10. checkpoint-100/special_tokens_map.json +5 -0
  11. checkpoint-100/target.spm +0 -0
  12. checkpoint-100/tokenizer_config.json +12 -0
  13. checkpoint-100/trainer_state.json +30 -0
  14. checkpoint-100/training_args.bin +3 -0
  15. checkpoint-100/vocab.json +0 -0
  16. checkpoint-200/added_tokens.json +44 -0
  17. checkpoint-200/config.json +61 -0
  18. checkpoint-200/generation_config.json +15 -0
  19. checkpoint-200/optimizer.pt +3 -0
  20. checkpoint-200/pytorch_model.bin +3 -0
  21. checkpoint-200/rng_state.pth +3 -0
  22. checkpoint-200/scaler.pt +3 -0
  23. checkpoint-200/scheduler.pt +3 -0
  24. checkpoint-200/source.spm +0 -0
  25. checkpoint-200/special_tokens_map.json +5 -0
  26. checkpoint-200/target.spm +0 -0
  27. checkpoint-200/tokenizer_config.json +12 -0
  28. checkpoint-200/trainer_state.json +44 -0
  29. checkpoint-200/training_args.bin +3 -0
  30. checkpoint-200/vocab.json +0 -0
checkpoint-100/added_tokens.json ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "|ach_aug_bt|": 64199,
3
+ "|ach_aug_ggl|": 64197,
4
+ "|ach_aug_hq|": 64198,
5
+ "|ach_bt|": 64196,
6
+ "|ach_ggl|": 64194,
7
+ "|ach_hq|": 64195,
8
+ "|ach_ood|": 64193,
9
+ "|lgg_aug_bt|": 64191,
10
+ "|lgg_aug_ggl|": 64190,
11
+ "|lgg_aug_hq|": 64192,
12
+ "|lgg_bt|": 64188,
13
+ "|lgg_ggl|": 64187,
14
+ "|lgg_hq|": 64189,
15
+ "|lgg_ood|": 64186,
16
+ "|lug_aug_bt|": 64206,
17
+ "|lug_aug_ggl|": 64204,
18
+ "|lug_aug_hq|": 64205,
19
+ "|lug_bt|": 64203,
20
+ "|lug_ggl|": 64201,
21
+ "|lug_hq|": 64202,
22
+ "|lug_ood|": 64200,
23
+ "|luo_aug_bt|": 64213,
24
+ "|luo_aug_ggl|": 64211,
25
+ "|luo_aug_hq|": 64212,
26
+ "|luo_bt|": 64210,
27
+ "|luo_ggl|": 64208,
28
+ "|luo_hq|": 64209,
29
+ "|luo_ood|": 64207,
30
+ "|nyn_aug_bt|": 64178,
31
+ "|nyn_aug_ggl|": 64176,
32
+ "|nyn_aug_hq|": 64177,
33
+ "|nyn_bt|": 64175,
34
+ "|nyn_ggl|": 64173,
35
+ "|nyn_hq|": 64174,
36
+ "|nyn_ood|": 64172,
37
+ "|teo_aug_bt|": 64185,
38
+ "|teo_aug_ggl|": 64183,
39
+ "|teo_aug_hq|": 64184,
40
+ "|teo_bt|": 64182,
41
+ "|teo_ggl|": 64180,
42
+ "|teo_hq|": 64181,
43
+ "|teo_ood|": 64179
44
+ }
checkpoint-100/config.json ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "Helsinki-NLP/opus-mt-mul-en",
3
+ "activation_dropout": 0.0,
4
+ "activation_function": "swish",
5
+ "add_bias_logits": false,
6
+ "add_final_layer_norm": false,
7
+ "architectures": [
8
+ "MarianMTModel"
9
+ ],
10
+ "attention_dropout": 0.0,
11
+ "bad_words_ids": [
12
+ [
13
+ 64171
14
+ ]
15
+ ],
16
+ "bos_token_id": 0,
17
+ "classif_dropout": 0.0,
18
+ "classifier_dropout": 0.0,
19
+ "d_model": 512,
20
+ "decoder_attention_heads": 8,
21
+ "decoder_ffn_dim": 2048,
22
+ "decoder_layerdrop": 0.0,
23
+ "decoder_layers": 6,
24
+ "decoder_start_token_id": 64171,
25
+ "decoder_vocab_size": 64214,
26
+ "dropout": 0.1,
27
+ "encoder_attention_heads": 8,
28
+ "encoder_ffn_dim": 2048,
29
+ "encoder_layerdrop": 0.0,
30
+ "encoder_layers": 6,
31
+ "eos_token_id": 0,
32
+ "extra_pos_embeddings": 64172,
33
+ "forced_eos_token_id": 0,
34
+ "id2label": {
35
+ "0": "LABEL_0",
36
+ "1": "LABEL_1",
37
+ "2": "LABEL_2"
38
+ },
39
+ "init_std": 0.02,
40
+ "is_encoder_decoder": true,
41
+ "label2id": {
42
+ "LABEL_0": 0,
43
+ "LABEL_1": 1,
44
+ "LABEL_2": 2
45
+ },
46
+ "max_length": 512,
47
+ "max_position_embeddings": 512,
48
+ "model_type": "marian",
49
+ "normalize_before": false,
50
+ "normalize_embedding": false,
51
+ "num_beams": 6,
52
+ "num_hidden_layers": 6,
53
+ "pad_token_id": 64171,
54
+ "scale_embedding": true,
55
+ "share_encoder_decoder_embeddings": true,
56
+ "static_position_embeddings": true,
57
+ "torch_dtype": "float32",
58
+ "transformers_version": "4.27.0",
59
+ "use_cache": true,
60
+ "vocab_size": 64214
61
+ }
checkpoint-100/generation_config.json ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bad_words_ids": [
3
+ [
4
+ 64171
5
+ ]
6
+ ],
7
+ "bos_token_id": 0,
8
+ "decoder_start_token_id": 64171,
9
+ "eos_token_id": 0,
10
+ "forced_eos_token_id": 0,
11
+ "max_length": 512,
12
+ "num_beams": 6,
13
+ "pad_token_id": 64171,
14
+ "transformers_version": "4.27.0"
15
+ }
checkpoint-100/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:eb6c22489befe3c46daa9a205afd7afcdc9e160cd9dc6a7cbd894189dadf9bb0
3
+ size 616276741
checkpoint-100/pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f895a2fc0e05014e0ac2bbf7dc09afd57787f1138c2edc73c82d73a3ec0732f0
3
+ size 308407621
checkpoint-100/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5f21c23323b5281789bb29ac46142bdde05833b39b8b271195f71f3eea11b501
3
+ size 14575
checkpoint-100/scaler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b8131f89c5cbaa717b917adf1f74e138486efeb837e12544a5f533c500fac84e
3
+ size 557
checkpoint-100/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:72985d69bf089688483feed17b52251afb5b2df3bbd9cffdbc67f45c1297b0a7
3
+ size 627
checkpoint-100/source.spm ADDED
Binary file (707 kB). View file
 
checkpoint-100/special_tokens_map.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "eos_token": "</s>",
3
+ "pad_token": "<pad>",
4
+ "unk_token": "<unk>"
5
+ }
checkpoint-100/target.spm ADDED
Binary file (791 kB). View file
 
checkpoint-100/tokenizer_config.json ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "eos_token": "</s>",
3
+ "model_max_length": 512,
4
+ "pad_token": "<pad>",
5
+ "separate_vocabs": false,
6
+ "source_lang": "mul",
7
+ "sp_model_kwargs": {},
8
+ "special_tokens_map_file": null,
9
+ "target_lang": "eng",
10
+ "tokenizer_class": "MarianTokenizer",
11
+ "unk_token": "<unk>"
12
+ }
checkpoint-100/trainer_state.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": 2.7249882221221924,
3
+ "best_model_checkpoint": "/home/azawahry_sunbird_ai/models/m2e/dataset-tags-expanded-tokenizer-oversampling/checkpoint-100",
4
+ "epoch": 0.1142369731771587,
5
+ "global_step": 100,
6
+ "is_hyper_param_search": false,
7
+ "is_local_process_zero": true,
8
+ "is_world_process_zero": true,
9
+ "log_history": [
10
+ {
11
+ "epoch": 0.11,
12
+ "eval_BLEU_ach": 0.0006,
13
+ "eval_BLEU_lgg": 0.0006,
14
+ "eval_BLEU_lug": 0.0006,
15
+ "eval_BLEU_mean": 0.0006,
16
+ "eval_BLEU_nyn": 0.0006,
17
+ "eval_BLEU_teo": 0.0006,
18
+ "eval_loss": 2.7249882221221924,
19
+ "eval_runtime": 1230.9404,
20
+ "eval_samples_per_second": 2.031,
21
+ "eval_steps_per_second": 0.051,
22
+ "step": 100
23
+ }
24
+ ],
25
+ "max_steps": 2625,
26
+ "num_train_epochs": 3,
27
+ "total_flos": 1.01695094784e+16,
28
+ "trial_name": null,
29
+ "trial_params": null
30
+ }
checkpoint-100/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0eaabea64c50e8627fb1a0d4d5af12cc6314c9de417fc2e6bc102a5d7bd00b0f
3
+ size 3707
checkpoint-100/vocab.json ADDED
The diff for this file is too large to render. See raw diff
 
checkpoint-200/added_tokens.json ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "|ach_aug_bt|": 64199,
3
+ "|ach_aug_ggl|": 64197,
4
+ "|ach_aug_hq|": 64198,
5
+ "|ach_bt|": 64196,
6
+ "|ach_ggl|": 64194,
7
+ "|ach_hq|": 64195,
8
+ "|ach_ood|": 64193,
9
+ "|lgg_aug_bt|": 64191,
10
+ "|lgg_aug_ggl|": 64190,
11
+ "|lgg_aug_hq|": 64192,
12
+ "|lgg_bt|": 64188,
13
+ "|lgg_ggl|": 64187,
14
+ "|lgg_hq|": 64189,
15
+ "|lgg_ood|": 64186,
16
+ "|lug_aug_bt|": 64206,
17
+ "|lug_aug_ggl|": 64204,
18
+ "|lug_aug_hq|": 64205,
19
+ "|lug_bt|": 64203,
20
+ "|lug_ggl|": 64201,
21
+ "|lug_hq|": 64202,
22
+ "|lug_ood|": 64200,
23
+ "|luo_aug_bt|": 64213,
24
+ "|luo_aug_ggl|": 64211,
25
+ "|luo_aug_hq|": 64212,
26
+ "|luo_bt|": 64210,
27
+ "|luo_ggl|": 64208,
28
+ "|luo_hq|": 64209,
29
+ "|luo_ood|": 64207,
30
+ "|nyn_aug_bt|": 64178,
31
+ "|nyn_aug_ggl|": 64176,
32
+ "|nyn_aug_hq|": 64177,
33
+ "|nyn_bt|": 64175,
34
+ "|nyn_ggl|": 64173,
35
+ "|nyn_hq|": 64174,
36
+ "|nyn_ood|": 64172,
37
+ "|teo_aug_bt|": 64185,
38
+ "|teo_aug_ggl|": 64183,
39
+ "|teo_aug_hq|": 64184,
40
+ "|teo_bt|": 64182,
41
+ "|teo_ggl|": 64180,
42
+ "|teo_hq|": 64181,
43
+ "|teo_ood|": 64179
44
+ }
checkpoint-200/config.json ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "Helsinki-NLP/opus-mt-mul-en",
3
+ "activation_dropout": 0.0,
4
+ "activation_function": "swish",
5
+ "add_bias_logits": false,
6
+ "add_final_layer_norm": false,
7
+ "architectures": [
8
+ "MarianMTModel"
9
+ ],
10
+ "attention_dropout": 0.0,
11
+ "bad_words_ids": [
12
+ [
13
+ 64171
14
+ ]
15
+ ],
16
+ "bos_token_id": 0,
17
+ "classif_dropout": 0.0,
18
+ "classifier_dropout": 0.0,
19
+ "d_model": 512,
20
+ "decoder_attention_heads": 8,
21
+ "decoder_ffn_dim": 2048,
22
+ "decoder_layerdrop": 0.0,
23
+ "decoder_layers": 6,
24
+ "decoder_start_token_id": 64171,
25
+ "decoder_vocab_size": 64214,
26
+ "dropout": 0.1,
27
+ "encoder_attention_heads": 8,
28
+ "encoder_ffn_dim": 2048,
29
+ "encoder_layerdrop": 0.0,
30
+ "encoder_layers": 6,
31
+ "eos_token_id": 0,
32
+ "extra_pos_embeddings": 64172,
33
+ "forced_eos_token_id": 0,
34
+ "id2label": {
35
+ "0": "LABEL_0",
36
+ "1": "LABEL_1",
37
+ "2": "LABEL_2"
38
+ },
39
+ "init_std": 0.02,
40
+ "is_encoder_decoder": true,
41
+ "label2id": {
42
+ "LABEL_0": 0,
43
+ "LABEL_1": 1,
44
+ "LABEL_2": 2
45
+ },
46
+ "max_length": 512,
47
+ "max_position_embeddings": 512,
48
+ "model_type": "marian",
49
+ "normalize_before": false,
50
+ "normalize_embedding": false,
51
+ "num_beams": 6,
52
+ "num_hidden_layers": 6,
53
+ "pad_token_id": 64171,
54
+ "scale_embedding": true,
55
+ "share_encoder_decoder_embeddings": true,
56
+ "static_position_embeddings": true,
57
+ "torch_dtype": "float32",
58
+ "transformers_version": "4.27.0",
59
+ "use_cache": true,
60
+ "vocab_size": 64214
61
+ }
checkpoint-200/generation_config.json ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bad_words_ids": [
3
+ [
4
+ 64171
5
+ ]
6
+ ],
7
+ "bos_token_id": 0,
8
+ "decoder_start_token_id": 64171,
9
+ "eos_token_id": 0,
10
+ "forced_eos_token_id": 0,
11
+ "max_length": 512,
12
+ "num_beams": 6,
13
+ "pad_token_id": 64171,
14
+ "transformers_version": "4.27.0"
15
+ }
checkpoint-200/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3cd8def39eb130a46ec34239e522ff7f729942532f7a495a2d4af58e470bb138
3
+ size 616276741
checkpoint-200/pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:04f60eb147a7c744617c80e8e9992ba98d77655250b55ba383b3c06fe2807055
3
+ size 308407621
checkpoint-200/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e941cec5d77ee9b8f69a2d89ccc95d86613ea069851879f4a3d3fe96e2361e5e
3
+ size 14575
checkpoint-200/scaler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7fac179f6b831faedfb0d0f64af3b143d0439f60a8a2c85513c58c3734a44b6e
3
+ size 557
checkpoint-200/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:152a51ef67cbf61b018ec24e90146d5e007a8853991f2e29156261e7c1d56914
3
+ size 627
checkpoint-200/source.spm ADDED
Binary file (707 kB). View file
 
checkpoint-200/special_tokens_map.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "eos_token": "</s>",
3
+ "pad_token": "<pad>",
4
+ "unk_token": "<unk>"
5
+ }
checkpoint-200/target.spm ADDED
Binary file (791 kB). View file
 
checkpoint-200/tokenizer_config.json ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "eos_token": "</s>",
3
+ "model_max_length": 512,
4
+ "pad_token": "<pad>",
5
+ "separate_vocabs": false,
6
+ "source_lang": "mul",
7
+ "sp_model_kwargs": {},
8
+ "special_tokens_map_file": null,
9
+ "target_lang": "eng",
10
+ "tokenizer_class": "MarianTokenizer",
11
+ "unk_token": "<unk>"
12
+ }
checkpoint-200/trainer_state.json ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": 2.7249882221221924,
3
+ "best_model_checkpoint": "/home/azawahry_sunbird_ai/models/m2e/dataset-tags-expanded-tokenizer-oversampling/checkpoint-100",
4
+ "epoch": 0.2284739463543174,
5
+ "global_step": 200,
6
+ "is_hyper_param_search": false,
7
+ "is_local_process_zero": true,
8
+ "is_world_process_zero": true,
9
+ "log_history": [
10
+ {
11
+ "epoch": 0.11,
12
+ "eval_BLEU_ach": 0.0006,
13
+ "eval_BLEU_lgg": 0.0006,
14
+ "eval_BLEU_lug": 0.0006,
15
+ "eval_BLEU_mean": 0.0006,
16
+ "eval_BLEU_nyn": 0.0006,
17
+ "eval_BLEU_teo": 0.0006,
18
+ "eval_loss": 2.7249882221221924,
19
+ "eval_runtime": 1230.9404,
20
+ "eval_samples_per_second": 2.031,
21
+ "eval_steps_per_second": 0.051,
22
+ "step": 100
23
+ },
24
+ {
25
+ "epoch": 0.23,
26
+ "eval_BLEU_ach": 0.0006,
27
+ "eval_BLEU_lgg": 0.0006,
28
+ "eval_BLEU_lug": 0.0006,
29
+ "eval_BLEU_mean": 0.0006,
30
+ "eval_BLEU_nyn": 0.0006,
31
+ "eval_BLEU_teo": 0.0006,
32
+ "eval_loss": 2.9588088989257812,
33
+ "eval_runtime": 1340.291,
34
+ "eval_samples_per_second": 1.865,
35
+ "eval_steps_per_second": 0.047,
36
+ "step": 200
37
+ }
38
+ ],
39
+ "max_steps": 2625,
40
+ "num_train_epochs": 3,
41
+ "total_flos": 2.03390189568e+16,
42
+ "trial_name": null,
43
+ "trial_params": null
44
+ }
checkpoint-200/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0eaabea64c50e8627fb1a0d4d5af12cc6314c9de417fc2e6bc102a5d7bd00b0f
3
+ size 3707
checkpoint-200/vocab.json ADDED
The diff for this file is too large to render. See raw diff