Initial commit

.gitattributes

@@ -0,0 +1,4 @@
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
+runs/** filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,129 @@
+---
+language:
+  - de
+tags:
+  - question-generation
+  - german
+  - text2text-generation
+  - generated_from_trainer
+datasets:
+  - lmqg/qg_dequad
+metrics:
+  - bleu4
+  - f1
+  - rouge
+  - exact_match
+model-index:
+  - name: german-jeopardy-mt5-large-256
+    results:
+      - task:
+          name: Sequence-to-sequence Language Modeling
+          type: text2text-generation
+        dataset:
+          name: lmqg/qg_dequad
+          type: default
+          args: default
+        metrics:
+          - name: BLEU-4
+            type: bleu4
+            value: 16.43
+          - name: F1
+            type: f1
+            value: 42.48
+          - name: ROUGE-1
+            type: rouge1
+            value: 43.56
+          - name: ROUGE-2
+            type: rouge2
+            value: 23.78
+          - name: ROUGE-L
+            type: rougel
+            value: 41.81
+          - name: ROUGE-Lsum
+            type: rougelsum
+            value: 41.80
+          - name: Exact Match
+            type: exact_match
+            value: 3.13
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# german-jeopardy-mt5-large-256
+
+This model is a fine-tuned version of [google/mt5-large](https://huggingface.co/google/mt5-large) on the [lmqg/qg_dequad](https://huggingface.co/datasets/lmqg/qg_dequad) dataset.
+It achieves the following results on the evaluation set:
+- Loss: 1.3943
+- Brevity Penalty: 0.9201
+- System Length: 19195
+- Reference Length: 20793
+- ROUGE-1: 43.56
+- ROUGE-2: 23.78
+- ROUGE-L: 41.81
+- ROUGE-Lsum: 41.80
+- Exact Match: 3.13
+- BLEU: 16.43
+- F1: 42.48
+
+## Model description
+
+
+See [google/mt5-large](https://huggingface.co/google/mt5-large) for the model architecture.  
+The model was trained on a single NVIDIA RTX 3090 GPU with 24GB of VRAM.
+
+## Intended uses & limitations
+
+This model can be used for question generation on German text.
+
+## Training and evaluation data
+
+See [lmqg/qg_dequad](https://huggingface.co/datasets/lmqg/qg_dequad).
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0001
+- train_batch_size: 1
+- eval_batch_size: 1
+- seed: 7
+- gradient_accumulation_steps: 256
+- total_train_batch_size: 256
+- optimizer: Adafactor
+- lr_scheduler_type: constant
+- num_epochs: 20
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Counts 1 | Counts 2 | Counts 3 | Counts 4 | Totals 1 | Totals 2 | Totals 3 | Totals 4 | Precisions 1 | Precisions 2 | Precisions 3 | Precisions 4 | Brevity Penalty | System Length | Reference Length | ROUGE-1 | ROUGE-2 | ROUGE-L | ROUGE-Lsum | Exact Match |  BLEU   | Mean Generated Length |   F1   |
+|:-------------:|:-----:|:----:|:---------------:|:--------:|:--------:|:--------:|:--------:|:--------:|:--------:|:--------:|:--------:|:------------:|:------------:|:------------:|:------------:|:---------------:|:-------------:|:----------------:|:-------:|:-------:|:-------:|:----------:|:-----------:|:-------:|:---------------------:|:------:|
+|     5.932     | 0.99  |  36  |     2.4510      |   5614   |   1426   |   527    |   204    |  28835   |  26631   |  24427   |  22223   |   19.4694    |    5.3547    |    2.1574    |    0.918     |       1.0       |     28835     |      21250       | 0.1946  | 0.0763  | 0.1843  |   0.1843   |     0.0     | 3.7906  |        11.4306        | 0.2127 |
+|    2.3089     | 1.98  |  72  |     1.3964      |   7578   |   2696   |   1244   |   580    |  17203   |  14999   |  12795   |  10591   |   44.0505    |   17.9745    |    9.7225    |    5.4763    |     0.7904      |     17203     |      21250       | 0.3312  | 0.1655  |  0.316  |   0.3162   |    0.01     | 11.3254 |        12.6583        | 0.3246 |
+|    1.6778     |  3.0  | 109  |     1.2660      |   7961   |   3020   |   1480   |   747    |  17067   |  14863   |  12659   |  10455   |   46.6456    |   20.3189    |   11.6913    |    7.1449    |     0.7826      |     17067     |      21250       | 0.3608  | 0.1881  | 0.3456  |   0.3454   |   0.0195    | 13.128  |        12.4682        | 0.3517 |
+|    1.5383     | 3.99  | 145  |     1.2212      |   7948   |   3121   |   1558   |   796    |  16694   |  14490   |  12286   |  10082   |   47.6099    |    21.539    |   12.6811    |    7.8953    |     0.7612      |     16694     |      21250       | 0.3663  | 0.1989  | 0.3523  |   0.352    |    0.024    | 13.625  |        12.221         | 0.3554 |
+|     1.423     | 4.97  | 181  |     1.1706      |   8746   |   3590   |   1840   |   963    |  17765   |  15561   |  13357   |  11153   |   49.2316    |   23.0705    |   13.7755    |    8.6344    |     0.8219      |     17765     |      21250       | 0.4033  | 0.2224  | 0.3876  |   0.3874   |   0.0304    | 15.7567 |        13.0277        | 0.3941 |
+|    1.2861     | 5.99  | 218  |     1.1327      |   8885   |   3646   |   1864   |   1005   |  17406   |  15202   |  12998   |  10794   |   51.0456    |   23.9837    |   14.3407    |    9.3107    |     0.8018      |     17406     |      21250       | 0.4181  | 0.2295  | 0.4022  |   0.402    |   0.0331    | 16.123  |        12.9142        | 0.4092 |
+|    1.2372     | 6.98  | 254  |     1.1248      |   9122   |   3824   |   1997   |   1084   |  17310   |  15106   |  12902   |  10698   |   52.6979    |   25.3144    |   15.4782    |   10.1327    |     0.7964      |     17310     |      21250       | 0.4313  |  0.239  | 0.4175  |   0.4172   |   0.0358    | 17.0334 |        12.8412        | 0.4236 |
+|    1.1307     |  8.0  | 291  |     1.0998      |   9423   |   4019   |   2136   |   1190   |  18074   |  15870   |  13666   |  11462   |   52.1357    |   25.3245    |    15.63     |   10.3821    |     0.8389      |     18074     |      21250       |  0.441  |  0.249  | 0.4255  |   0.4252   |   0.0404    | 18.0474 |        13.4138        | 0.4327 |
+|    1.0982     | 8.99  | 327  |     1.1052      |   9450   |   4003   |   2147   |   1184   |  18145   |  15941   |  13737   |  11533   |   52.0805    |   25.1113    |   15.6293    |   10.2662    |     0.8427      |     18145     |      21250       | 0.4427  | 0.2492  | 0.4266  |   0.4261   |   0.0426    | 18.0367 |        13.4465        | 0.4344 |
+|    1.0449     | 9.98  | 363  |     1.0996      |   9471   |   4036   |   2149   |   1180   |  18067   |  15863   |  13659   |  11455   |   52.4215    |   25.4429    |   15.7332    |   10.3012    |     0.8385      |     18067     |      21250       | 0.4422  | 0.2477  | 0.4261  |   0.4257   |   0.0404    | 18.0793 |        13.333         | 0.4341 |
+|    0.9686     | 10.99 | 400  |     1.1012      |   9612   |   4165   |   2240   |   1233   |  17983   |  15779   |  13575   |  11371   |   53.4505    |   26.3958    |   16.5009    |   10.8434    |     0.8339      |     17983     |      21250       | 0.4534  | 0.2591  | 0.4381  |   0.4378   |   0.0449    | 18.6914 |        13.3534        | 0.4458 |
+|    0.9465     | 11.98 | 436  |     1.1027      |   9670   |   4154   |   2229   |   1239   |  18217   |  16013   |  13809   |  11605   |   53.0823    |   25.9414    |   16.1416    |   10.6764    |     0.8466      |     18217     |      21250       | 0.4531  |  0.258  | 0.4377  |   0.4374   |   0.0445    | 18.6863 |        13.5912        | 0.4452 |
+|    0.9025     | 12.97 | 472  |     1.1124      |   9627   |   4155   |   2241   |   1247   |  18076   |  15872   |  13668   |  11464   |   53.2585    |   26.1782    |    16.396    |   10.8775    |      0.839      |     18076     |      21250       | 0.4531  | 0.2583  | 0.4386  |   0.4382   |   0.0436    | 18.7344 |        13.5259        | 0.4452 |
+|    0.8402     | 13.99 | 509  |     1.1392      |   9425   |   4071   |   2176   |   1207   |  17339   |  15135   |  12931   |  10727   |   54.3572    |   26.8979    |   16.8278    |    11.252    |     0.7981      |     17339     |      21250       | 0.4495  | 0.2568  | 0.4365  |   0.4358   |   0.0445    | 18.3062 |        12.9129        | 0.4417 |
+|    0.8282     | 14.98 | 545  |     1.1227      |   9803   |   4274   |   2316   |   1305   |  18652   |  16448   |  14244   |  12040   |   52.5574    |   25.9849    |   16.2595    |   10.8389    |      0.87       |     18652     |      21250       | 0.4573  | 0.2627  | 0.4418  |   0.4414   |   0.0463    | 19.2695 |        14.0104        | 0.4496 |
+|    0.7694     | 16.0  | 582  |     1.1394      |   9740   |   4240   |   2299   |   1296   |  18281   |  16077   |  13873   |  11669   |   53.2794    |   26.3731    |   16.5718    |   11.1064    |     0.8501      |     18281     |      21250       | 0.4572  | 0.2629  | 0.4411  |   0.4412   |   0.0476    | 19.1704 |        13.6475        | 0.4492 |
+|    0.7589     | 16.99 | 618  |     1.1497      |   9663   |   4140   |   2214   |   1232   |  18412   |  16208   |  14004   |  11800   |   52.4821    |   25.5429    |   15.8098    |   10.4407    |     0.8572      |     18412     |      21250       | 0.4515  | 0.2561  | 0.4359  |   0.4358   |    0.044    | 18.5906 |        13.7926        | 0.4432 |
+|     0.724     | 17.98 | 654  |     1.1680      |   9743   |   4246   |   2316   |   1300   |  18402   |  16198   |  13994   |  11790   |   52.9453    |   26.2131    |   16.5499    |   11.0263    |     0.8566      |     18402     |      21250       | 0.4562  | 0.2625  | 0.4408  |   0.441    |   0.0472    | 19.2167 |        13.7214        | 0.4474 |
+|    0.6755     | 18.99 | 691  |     1.1874      |   9722   |   4266   |   2351   |   1341   |  18272   |  16068   |  13864   |  11660   |   53.2071    |   26.5497    |   16.9576    |   11.5009    |     0.8496      |     18272     |      21250       | 0.4559  | 0.2639  | 0.4417  |   0.4413   |   0.0495    | 19.4647 |        13.6071        | 0.4469 |
+|     0.657     | 19.79 | 720  |     1.1845      |   9920   |   4361   |   2402   |   1373   |  18884   |  16680   |  14476   |  12272   |   52.5312    |   26.1451    |    16.593    |   11.1881    |     0.8822      |     18884     |      21250       | 0.4594  | 0.2647  | 0.4423  |   0.4421   |   0.0467    | 19.8248 |        14.2001        | 0.4508 |
+
+
+### Framework versions
+
+- Transformers 4.32.1
+- Pytorch 2.1.0
+- Datasets 2.12.0
+- Tokenizers 0.13.3

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<hl>": 250100
+}

all_results.json

@@ -0,0 +1,64 @@
+{
+    "epoch": 19.79,
+    "eval_bleu": 19.8248,
+    "eval_bp": 0.8822,
+    "eval_counts_1": 9920,
+    "eval_counts_2": 4361,
+    "eval_counts_3": 2402,
+    "eval_counts_4": 1373,
+    "eval_exact_match": 0.0467,
+    "eval_f1": 0.4508,
+    "eval_gen_len": 14.2001,
+    "eval_loss": 1.1845453977584839,
+    "eval_precisions_1": 52.5312,
+    "eval_precisions_2": 26.1451,
+    "eval_precisions_3": 16.593,
+    "eval_precisions_4": 11.1881,
+    "eval_ref_len": 21250,
+    "eval_rouge1": 0.4594,
+    "eval_rouge2": 0.2647,
+    "eval_rougeL": 0.4423,
+    "eval_rougeLsum": 0.4421,
+    "eval_runtime": 2997.7936,
+    "eval_samples": 2204,
+    "eval_samples_per_second": 0.735,
+    "eval_steps_per_second": 0.735,
+    "eval_sys_len": 18884,
+    "eval_totals_1": 18884,
+    "eval_totals_2": 16680,
+    "eval_totals_3": 14476,
+    "eval_totals_4": 12272,
+    "predict_bleu": 15.9031,
+    "predict_bp": 0.9242,
+    "predict_counts_1": 9231,
+    "predict_counts_2": 3627,
+    "predict_counts_3": 1778,
+    "predict_counts_4": 912,
+    "predict_exact_match": 0.0304,
+    "predict_f1": 0.4153,
+    "predict_gen_len": 14.6175,
+    "predict_loss": 1.4399399757385254,
+    "predict_precisions_1": 47.896,
+    "predict_precisions_2": 21.249,
+    "predict_precisions_3": 11.961,
+    "predict_precisions_4": 7.2032,
+    "predict_ref_len": 20793,
+    "predict_rouge1": 0.4255,
+    "predict_rouge2": 0.2307,
+    "predict_rougeL": 0.4079,
+    "predict_rougeLsum": 0.408,
+    "predict_runtime": 2027.9694,
+    "predict_samples": 2204,
+    "predict_samples_per_second": 1.087,
+    "predict_steps_per_second": 1.087,
+    "predict_sys_len": 19273,
+    "predict_totals_1": 19273,
+    "predict_totals_2": 17069,
+    "predict_totals_3": 14865,
+    "predict_totals_4": 12661,
+    "train_loss": 1.3412119759453667,
+    "train_runtime": 128063.7495,
+    "train_samples": 9314,
+    "train_samples_per_second": 1.455,
+    "train_steps_per_second": 0.006
+}

config.json

@@ -0,0 +1,36 @@
+{
+  "_name_or_path": "google/mt5-large",
+  "architectures": [
+    "MT5ForConditionalGeneration"
+  ],
+  "classifier_dropout": 0.0,
+  "d_ff": 2816,
+  "d_kv": 64,
+  "d_model": 1024,
+  "decoder_start_token_id": 0,
+  "dense_act_fn": "gelu_new",
+  "dropout_rate": 0.1,
+  "eos_token_id": 1,
+  "feed_forward_proj": "gated-gelu",
+  "initializer_factor": 1.0,
+  "is_encoder_decoder": true,
+  "is_gated_act": true,
+  "layer_norm_epsilon": 1e-06,
+  "length_penalty": 0.0,
+  "max_length": 64,
+  "model_type": "mt5",
+  "num_beams": 4,
+  "num_decoder_layers": 24,
+  "num_heads": 16,
+  "num_layers": 24,
+  "output_past": true,
+  "pad_token_id": 0,
+  "relative_attention_max_distance": 128,
+  "relative_attention_num_buckets": 32,
+  "tie_word_embeddings": false,
+  "tokenizer_class": "T5Tokenizer",
+  "torch_dtype": "float32",
+  "transformers_version": "4.32.1",
+  "use_cache": true,
+  "vocab_size": 250112
+}

generation_config.json

@@ -0,0 +1,9 @@
+{
+  "decoder_start_token_id": 0,
+  "eos_token_id": 1,
+  "length_penalty": 0.0,
+  "max_length": 64,
+  "num_beams": 4,
+  "pad_token_id": 0,
+  "transformers_version": "4.32.1"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0046fba082045672b0afcf4bdb7540c6c186e9ab9136ef77b08f18b67134697b
+size 4918393736

special_tokens_map.json

@@ -0,0 +1,8 @@
+{
+  "additional_special_tokens": [
+    "<hl>"
+  ],
+  "eos_token": "</s>",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
+}

spiece.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef78f86560d809067d12bac6c09f19a462cb3af3f54d2b8acbba26e1433125d6
+size 4309802

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ef4273f90fa3341fdb81c66eb7973b1651a8183e57c8ab9e9144aac61b48f9f
+size 16330550

tokenizer_config.json

@@ -0,0 +1,12 @@
+{
+  "additional_special_tokens": null,
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "</s>",
+  "extra_ids": 0,
+  "legacy": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "T5Tokenizer",
+  "unk_token": "<unk>"
+}

trainer_state.json

@@ -0,0 +1,768 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 19.789564097058193,
+  "eval_steps": 500,
+  "global_step": 720,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.99,
+      "learning_rate": 0.0001,
+      "loss": 5.932,
+      "step": 36
+    },
+    {
+      "epoch": 0.99,
+      "eval_bleu": 3.7906,
+      "eval_bp": 1.0,
+      "eval_counts_1": 5614,
+      "eval_counts_2": 1426,
+      "eval_counts_3": 527,
+      "eval_counts_4": 204,
+      "eval_exact_match": 0.0,
+      "eval_f1": 0.2127,
+      "eval_gen_len": 11.4306,
+      "eval_loss": 2.450988292694092,
+      "eval_precisions_1": 19.4694,
+      "eval_precisions_2": 5.3547,
+      "eval_precisions_3": 2.1574,
+      "eval_precisions_4": 0.918,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.1946,
+      "eval_rouge2": 0.0763,
+      "eval_rougeL": 0.1843,
+      "eval_rougeLsum": 0.1843,
+      "eval_runtime": 1846.0551,
+      "eval_samples_per_second": 1.194,
+      "eval_steps_per_second": 1.194,
+      "eval_sys_len": 28835,
+      "eval_totals_1": 28835,
+      "eval_totals_2": 26631,
+      "eval_totals_3": 24427,
+      "eval_totals_4": 22223,
+      "step": 36
+    },
+    {
+      "epoch": 1.98,
+      "learning_rate": 0.0001,
+      "loss": 2.3089,
+      "step": 72
+    },
+    {
+      "epoch": 1.98,
+      "eval_bleu": 11.3254,
+      "eval_bp": 0.7904,
+      "eval_counts_1": 7578,
+      "eval_counts_2": 2696,
+      "eval_counts_3": 1244,
+      "eval_counts_4": 580,
+      "eval_exact_match": 0.01,
+      "eval_f1": 0.3246,
+      "eval_gen_len": 12.6583,
+      "eval_loss": 1.3963948488235474,
+      "eval_precisions_1": 44.0505,
+      "eval_precisions_2": 17.9745,
+      "eval_precisions_3": 9.7225,
+      "eval_precisions_4": 5.4763,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.3312,
+      "eval_rouge2": 0.1655,
+      "eval_rougeL": 0.316,
+      "eval_rougeLsum": 0.3162,
+      "eval_runtime": 1912.8668,
+      "eval_samples_per_second": 1.152,
+      "eval_steps_per_second": 1.152,
+      "eval_sys_len": 17203,
+      "eval_totals_1": 17203,
+      "eval_totals_2": 14999,
+      "eval_totals_3": 12795,
+      "eval_totals_4": 10591,
+      "step": 72
+    },
+    {
+      "epoch": 3.0,
+      "learning_rate": 0.0001,
+      "loss": 1.6778,
+      "step": 109
+    },
+    {
+      "epoch": 3.0,
+      "eval_bleu": 13.128,
+      "eval_bp": 0.7826,
+      "eval_counts_1": 7961,
+      "eval_counts_2": 3020,
+      "eval_counts_3": 1480,
+      "eval_counts_4": 747,
+      "eval_exact_match": 0.0195,
+      "eval_f1": 0.3517,
+      "eval_gen_len": 12.4682,
+      "eval_loss": 1.2659858465194702,
+      "eval_precisions_1": 46.6456,
+      "eval_precisions_2": 20.3189,
+      "eval_precisions_3": 11.6913,
+      "eval_precisions_4": 7.1449,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.3608,
+      "eval_rouge2": 0.1881,
+      "eval_rougeL": 0.3456,
+      "eval_rougeLsum": 0.3454,
+      "eval_runtime": 1864.6637,
+      "eval_samples_per_second": 1.182,
+      "eval_steps_per_second": 1.182,
+      "eval_sys_len": 17067,
+      "eval_totals_1": 17067,
+      "eval_totals_2": 14863,
+      "eval_totals_3": 12659,
+      "eval_totals_4": 10455,
+      "step": 109
+    },
+    {
+      "epoch": 3.99,
+      "learning_rate": 0.0001,
+      "loss": 1.5383,
+      "step": 145
+    },
+    {
+      "epoch": 3.99,
+      "eval_bleu": 13.625,
+      "eval_bp": 0.7612,
+      "eval_counts_1": 7948,
+      "eval_counts_2": 3121,
+      "eval_counts_3": 1558,
+      "eval_counts_4": 796,
+      "eval_exact_match": 0.024,
+      "eval_f1": 0.3554,
+      "eval_gen_len": 12.221,
+      "eval_loss": 1.2212449312210083,
+      "eval_precisions_1": 47.6099,
+      "eval_precisions_2": 21.539,
+      "eval_precisions_3": 12.6811,
+      "eval_precisions_4": 7.8953,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.3663,
+      "eval_rouge2": 0.1989,
+      "eval_rougeL": 0.3523,
+      "eval_rougeLsum": 0.352,
+      "eval_runtime": 1862.6167,
+      "eval_samples_per_second": 1.183,
+      "eval_steps_per_second": 1.183,
+      "eval_sys_len": 16694,
+      "eval_totals_1": 16694,
+      "eval_totals_2": 14490,
+      "eval_totals_3": 12286,
+      "eval_totals_4": 10082,
+      "step": 145
+    },
+    {
+      "epoch": 4.97,
+      "learning_rate": 0.0001,
+      "loss": 1.423,
+      "step": 181
+    },
+    {
+      "epoch": 4.97,
+      "eval_bleu": 15.7567,
+      "eval_bp": 0.8219,
+      "eval_counts_1": 8746,
+      "eval_counts_2": 3590,
+      "eval_counts_3": 1840,
+      "eval_counts_4": 963,
+      "eval_exact_match": 0.0304,
+      "eval_f1": 0.3941,
+      "eval_gen_len": 13.0277,
+      "eval_loss": 1.1706066131591797,
+      "eval_precisions_1": 49.2316,
+      "eval_precisions_2": 23.0705,
+      "eval_precisions_3": 13.7755,
+      "eval_precisions_4": 8.6344,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.4033,
+      "eval_rouge2": 0.2224,
+      "eval_rougeL": 0.3876,
+      "eval_rougeLsum": 0.3874,
+      "eval_runtime": 1923.5708,
+      "eval_samples_per_second": 1.146,
+      "eval_steps_per_second": 1.146,
+      "eval_sys_len": 17765,
+      "eval_totals_1": 17765,
+      "eval_totals_2": 15561,
+      "eval_totals_3": 13357,
+      "eval_totals_4": 11153,
+      "step": 181
+    },
+    {
+      "epoch": 5.99,
+      "learning_rate": 0.0001,
+      "loss": 1.2861,
+      "step": 218
+    },
+    {
+      "epoch": 5.99,
+      "eval_bleu": 16.123,
+      "eval_bp": 0.8018,
+      "eval_counts_1": 8885,
+      "eval_counts_2": 3646,
+      "eval_counts_3": 1864,
+      "eval_counts_4": 1005,
+      "eval_exact_match": 0.0331,
+      "eval_f1": 0.4092,
+      "eval_gen_len": 12.9142,
+      "eval_loss": 1.1327459812164307,
+      "eval_precisions_1": 51.0456,
+      "eval_precisions_2": 23.9837,
+      "eval_precisions_3": 14.3407,
+      "eval_precisions_4": 9.3107,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.4181,
+      "eval_rouge2": 0.2295,
+      "eval_rougeL": 0.4022,
+      "eval_rougeLsum": 0.402,
+      "eval_runtime": 2271.0986,
+      "eval_samples_per_second": 0.97,
+      "eval_steps_per_second": 0.97,
+      "eval_sys_len": 17406,
+      "eval_totals_1": 17406,
+      "eval_totals_2": 15202,
+      "eval_totals_3": 12998,
+      "eval_totals_4": 10794,
+      "step": 218
+    },
+    {
+      "epoch": 6.98,
+      "learning_rate": 0.0001,
+      "loss": 1.2372,
+      "step": 254
+    },
+    {
+      "epoch": 6.98,
+      "eval_bleu": 17.0334,
+      "eval_bp": 0.7964,
+      "eval_counts_1": 9122,
+      "eval_counts_2": 3824,
+      "eval_counts_3": 1997,
+      "eval_counts_4": 1084,
+      "eval_exact_match": 0.0358,
+      "eval_f1": 0.4236,
+      "eval_gen_len": 12.8412,
+      "eval_loss": 1.1248232126235962,
+      "eval_precisions_1": 52.6979,
+      "eval_precisions_2": 25.3144,
+      "eval_precisions_3": 15.4782,
+      "eval_precisions_4": 10.1327,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.4313,
+      "eval_rouge2": 0.239,
+      "eval_rougeL": 0.4175,
+      "eval_rougeLsum": 0.4172,
+      "eval_runtime": 2069.7482,
+      "eval_samples_per_second": 1.065,
+      "eval_steps_per_second": 1.065,
+      "eval_sys_len": 17310,
+      "eval_totals_1": 17310,
+      "eval_totals_2": 15106,
+      "eval_totals_3": 12902,
+      "eval_totals_4": 10698,
+      "step": 254
+    },
+    {
+      "epoch": 8.0,
+      "learning_rate": 0.0001,
+      "loss": 1.1307,
+      "step": 291
+    },
+    {
+      "epoch": 8.0,
+      "eval_bleu": 18.0474,
+      "eval_bp": 0.8389,
+      "eval_counts_1": 9423,
+      "eval_counts_2": 4019,
+      "eval_counts_3": 2136,
+      "eval_counts_4": 1190,
+      "eval_exact_match": 0.0404,
+      "eval_f1": 0.4327,
+      "eval_gen_len": 13.4138,
+      "eval_loss": 1.0998262166976929,
+      "eval_precisions_1": 52.1357,
+      "eval_precisions_2": 25.3245,
+      "eval_precisions_3": 15.63,
+      "eval_precisions_4": 10.3821,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.441,
+      "eval_rouge2": 0.249,
+      "eval_rougeL": 0.4255,
+      "eval_rougeLsum": 0.4252,
+      "eval_runtime": 2305.9504,
+      "eval_samples_per_second": 0.956,
+      "eval_steps_per_second": 0.956,
+      "eval_sys_len": 18074,
+      "eval_totals_1": 18074,
+      "eval_totals_2": 15870,
+      "eval_totals_3": 13666,
+      "eval_totals_4": 11462,
+      "step": 291
+    },
+    {
+      "epoch": 8.99,
+      "learning_rate": 0.0001,
+      "loss": 1.0982,
+      "step": 327
+    },
+    {
+      "epoch": 8.99,
+      "eval_bleu": 18.0367,
+      "eval_bp": 0.8427,
+      "eval_counts_1": 9450,
+      "eval_counts_2": 4003,
+      "eval_counts_3": 2147,
+      "eval_counts_4": 1184,
+      "eval_exact_match": 0.0426,
+      "eval_f1": 0.4344,
+      "eval_gen_len": 13.4465,
+      "eval_loss": 1.1051570177078247,
+      "eval_precisions_1": 52.0805,
+      "eval_precisions_2": 25.1113,
+      "eval_precisions_3": 15.6293,
+      "eval_precisions_4": 10.2662,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.4427,
+      "eval_rouge2": 0.2492,
+      "eval_rougeL": 0.4266,
+      "eval_rougeLsum": 0.4261,
+      "eval_runtime": 2511.4033,
+      "eval_samples_per_second": 0.878,
+      "eval_steps_per_second": 0.878,
+      "eval_sys_len": 18145,
+      "eval_totals_1": 18145,
+      "eval_totals_2": 15941,
+      "eval_totals_3": 13737,
+      "eval_totals_4": 11533,
+      "step": 327
+    },
+    {
+      "epoch": 9.98,
+      "learning_rate": 0.0001,
+      "loss": 1.0449,
+      "step": 363
+    },
+    {
+      "epoch": 9.98,
+      "eval_bleu": 18.0793,
+      "eval_bp": 0.8385,
+      "eval_counts_1": 9471,
+      "eval_counts_2": 4036,
+      "eval_counts_3": 2149,
+      "eval_counts_4": 1180,
+      "eval_exact_match": 0.0404,
+      "eval_f1": 0.4341,
+      "eval_gen_len": 13.333,
+      "eval_loss": 1.0995820760726929,
+      "eval_precisions_1": 52.4215,
+      "eval_precisions_2": 25.4429,
+      "eval_precisions_3": 15.7332,
+      "eval_precisions_4": 10.3012,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.4422,
+      "eval_rouge2": 0.2477,
+      "eval_rougeL": 0.4261,
+      "eval_rougeLsum": 0.4257,
+      "eval_runtime": 1973.5312,
+      "eval_samples_per_second": 1.117,
+      "eval_steps_per_second": 1.117,
+      "eval_sys_len": 18067,
+      "eval_totals_1": 18067,
+      "eval_totals_2": 15863,
+      "eval_totals_3": 13659,
+      "eval_totals_4": 11455,
+      "step": 363
+    },
+    {
+      "epoch": 10.99,
+      "learning_rate": 0.0001,
+      "loss": 0.9686,
+      "step": 400
+    },
+    {
+      "epoch": 10.99,
+      "eval_bleu": 18.6914,
+      "eval_bp": 0.8339,
+      "eval_counts_1": 9612,
+      "eval_counts_2": 4165,
+      "eval_counts_3": 2240,
+      "eval_counts_4": 1233,
+      "eval_exact_match": 0.0449,
+      "eval_f1": 0.4458,
+      "eval_gen_len": 13.3534,
+      "eval_loss": 1.1012390851974487,
+      "eval_precisions_1": 53.4505,
+      "eval_precisions_2": 26.3958,
+      "eval_precisions_3": 16.5009,
+      "eval_precisions_4": 10.8434,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.4534,
+      "eval_rouge2": 0.2591,
+      "eval_rougeL": 0.4381,
+      "eval_rougeLsum": 0.4378,
+      "eval_runtime": 2028.4468,
+      "eval_samples_per_second": 1.087,
+      "eval_steps_per_second": 1.087,
+      "eval_sys_len": 17983,
+      "eval_totals_1": 17983,
+      "eval_totals_2": 15779,
+      "eval_totals_3": 13575,
+      "eval_totals_4": 11371,
+      "step": 400
+    },
+    {
+      "epoch": 11.98,
+      "learning_rate": 0.0001,
+      "loss": 0.9465,
+      "step": 436
+    },
+    {
+      "epoch": 11.98,
+      "eval_bleu": 18.6863,
+      "eval_bp": 0.8466,
+      "eval_counts_1": 9670,
+      "eval_counts_2": 4154,
+      "eval_counts_3": 2229,
+      "eval_counts_4": 1239,
+      "eval_exact_match": 0.0445,
+      "eval_f1": 0.4452,
+      "eval_gen_len": 13.5912,
+      "eval_loss": 1.1026833057403564,
+      "eval_precisions_1": 53.0823,
+      "eval_precisions_2": 25.9414,
+      "eval_precisions_3": 16.1416,
+      "eval_precisions_4": 10.6764,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.4531,
+      "eval_rouge2": 0.258,
+      "eval_rougeL": 0.4377,
+      "eval_rougeLsum": 0.4374,
+      "eval_runtime": 4413.1713,
+      "eval_samples_per_second": 0.499,
+      "eval_steps_per_second": 0.499,
+      "eval_sys_len": 18217,
+      "eval_totals_1": 18217,
+      "eval_totals_2": 16013,
+      "eval_totals_3": 13809,
+      "eval_totals_4": 11605,
+      "step": 436
+    },
+    {
+      "epoch": 12.97,
+      "learning_rate": 0.0001,
+      "loss": 0.9025,
+      "step": 472
+    },
+    {
+      "epoch": 12.97,
+      "eval_bleu": 18.7344,
+      "eval_bp": 0.839,
+      "eval_counts_1": 9627,
+      "eval_counts_2": 4155,
+      "eval_counts_3": 2241,
+      "eval_counts_4": 1247,
+      "eval_exact_match": 0.0436,
+      "eval_f1": 0.4452,
+      "eval_gen_len": 13.5259,
+      "eval_loss": 1.11244535446167,
+      "eval_precisions_1": 53.2585,
+      "eval_precisions_2": 26.1782,
+      "eval_precisions_3": 16.396,
+      "eval_precisions_4": 10.8775,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.4531,
+      "eval_rouge2": 0.2583,
+      "eval_rougeL": 0.4386,
+      "eval_rougeLsum": 0.4382,
+      "eval_runtime": 3852.502,
+      "eval_samples_per_second": 0.572,
+      "eval_steps_per_second": 0.572,
+      "eval_sys_len": 18076,
+      "eval_totals_1": 18076,
+      "eval_totals_2": 15872,
+      "eval_totals_3": 13668,
+      "eval_totals_4": 11464,
+      "step": 472
+    },
+    {
+      "epoch": 13.99,
+      "learning_rate": 0.0001,
+      "loss": 0.8402,
+      "step": 509
+    },
+    {
+      "epoch": 13.99,
+      "eval_bleu": 18.3062,
+      "eval_bp": 0.7981,
+      "eval_counts_1": 9425,
+      "eval_counts_2": 4071,
+      "eval_counts_3": 2176,
+      "eval_counts_4": 1207,
+      "eval_exact_match": 0.0445,
+      "eval_f1": 0.4417,
+      "eval_gen_len": 12.9129,
+      "eval_loss": 1.139233946800232,
+      "eval_precisions_1": 54.3572,
+      "eval_precisions_2": 26.8979,
+      "eval_precisions_3": 16.8278,
+      "eval_precisions_4": 11.252,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.4495,
+      "eval_rouge2": 0.2568,
+      "eval_rougeL": 0.4365,
+      "eval_rougeLsum": 0.4358,
+      "eval_runtime": 3679.9149,
+      "eval_samples_per_second": 0.599,
+      "eval_steps_per_second": 0.599,
+      "eval_sys_len": 17339,
+      "eval_totals_1": 17339,
+      "eval_totals_2": 15135,
+      "eval_totals_3": 12931,
+      "eval_totals_4": 10727,
+      "step": 509
+    },
+    {
+      "epoch": 14.98,
+      "learning_rate": 0.0001,
+      "loss": 0.8282,
+      "step": 545
+    },
+    {
+      "epoch": 14.98,
+      "eval_bleu": 19.2695,
+      "eval_bp": 0.87,
+      "eval_counts_1": 9803,
+      "eval_counts_2": 4274,
+      "eval_counts_3": 2316,
+      "eval_counts_4": 1305,
+      "eval_exact_match": 0.0463,
+      "eval_f1": 0.4496,
+      "eval_gen_len": 14.0104,
+      "eval_loss": 1.1227205991744995,
+      "eval_precisions_1": 52.5574,
+      "eval_precisions_2": 25.9849,
+      "eval_precisions_3": 16.2595,
+      "eval_precisions_4": 10.8389,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.4573,
+      "eval_rouge2": 0.2627,
+      "eval_rougeL": 0.4418,
+      "eval_rougeLsum": 0.4414,
+      "eval_runtime": 3897.0455,
+      "eval_samples_per_second": 0.566,
+      "eval_steps_per_second": 0.566,
+      "eval_sys_len": 18652,
+      "eval_totals_1": 18652,
+      "eval_totals_2": 16448,
+      "eval_totals_3": 14244,
+      "eval_totals_4": 12040,
+      "step": 545
+    },
+    {
+      "epoch": 16.0,
+      "learning_rate": 0.0001,
+      "loss": 0.7694,
+      "step": 582
+    },
+    {
+      "epoch": 16.0,
+      "eval_bleu": 19.1704,
+      "eval_bp": 0.8501,
+      "eval_counts_1": 9740,
+      "eval_counts_2": 4240,
+      "eval_counts_3": 2299,
+      "eval_counts_4": 1296,
+      "eval_exact_match": 0.0476,
+      "eval_f1": 0.4492,
+      "eval_gen_len": 13.6475,
+      "eval_loss": 1.139431357383728,
+      "eval_precisions_1": 53.2794,
+      "eval_precisions_2": 26.3731,
+      "eval_precisions_3": 16.5718,
+      "eval_precisions_4": 11.1064,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.4572,
+      "eval_rouge2": 0.2629,
+      "eval_rougeL": 0.4411,
+      "eval_rougeLsum": 0.4412,
+      "eval_runtime": 3879.8331,
+      "eval_samples_per_second": 0.568,
+      "eval_steps_per_second": 0.568,
+      "eval_sys_len": 18281,
+      "eval_totals_1": 18281,
+      "eval_totals_2": 16077,
+      "eval_totals_3": 13873,
+      "eval_totals_4": 11669,
+      "step": 582
+    },
+    {
+      "epoch": 16.99,
+      "learning_rate": 0.0001,
+      "loss": 0.7589,
+      "step": 618
+    },
+    {
+      "epoch": 16.99,
+      "eval_bleu": 18.5906,
+      "eval_bp": 0.8572,
+      "eval_counts_1": 9663,
+      "eval_counts_2": 4140,
+      "eval_counts_3": 2214,
+      "eval_counts_4": 1232,
+      "eval_exact_match": 0.044,
+      "eval_f1": 0.4432,
+      "eval_gen_len": 13.7926,
+      "eval_loss": 1.1496515274047852,
+      "eval_precisions_1": 52.4821,
+      "eval_precisions_2": 25.5429,
+      "eval_precisions_3": 15.8098,
+      "eval_precisions_4": 10.4407,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.4515,
+      "eval_rouge2": 0.2561,
+      "eval_rougeL": 0.4359,
+      "eval_rougeLsum": 0.4358,
+      "eval_runtime": 3896.3752,
+      "eval_samples_per_second": 0.566,
+      "eval_steps_per_second": 0.566,
+      "eval_sys_len": 18412,
+      "eval_totals_1": 18412,
+      "eval_totals_2": 16208,
+      "eval_totals_3": 14004,
+      "eval_totals_4": 11800,
+      "step": 618
+    },
+    {
+      "epoch": 17.98,
+      "learning_rate": 0.0001,
+      "loss": 0.724,
+      "step": 654
+    },
+    {
+      "epoch": 17.98,
+      "eval_bleu": 19.2167,
+      "eval_bp": 0.8566,
+      "eval_counts_1": 9743,
+      "eval_counts_2": 4246,
+      "eval_counts_3": 2316,
+      "eval_counts_4": 1300,
+      "eval_exact_match": 0.0472,
+      "eval_f1": 0.4474,
+      "eval_gen_len": 13.7214,
+      "eval_loss": 1.1680002212524414,
+      "eval_precisions_1": 52.9453,
+      "eval_precisions_2": 26.2131,
+      "eval_precisions_3": 16.5499,
+      "eval_precisions_4": 11.0263,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.4562,
+      "eval_rouge2": 0.2625,
+      "eval_rougeL": 0.4408,
+      "eval_rougeLsum": 0.441,
+      "eval_runtime": 3957.7799,
+      "eval_samples_per_second": 0.557,
+      "eval_steps_per_second": 0.557,
+      "eval_sys_len": 18402,
+      "eval_totals_1": 18402,
+      "eval_totals_2": 16198,
+      "eval_totals_3": 13994,
+      "eval_totals_4": 11790,
+      "step": 654
+    },
+    {
+      "epoch": 18.99,
+      "learning_rate": 0.0001,
+      "loss": 0.6755,
+      "step": 691
+    },
+    {
+      "epoch": 18.99,
+      "eval_bleu": 19.4647,
+      "eval_bp": 0.8496,
+      "eval_counts_1": 9722,
+      "eval_counts_2": 4266,
+      "eval_counts_3": 2351,
+      "eval_counts_4": 1341,
+      "eval_exact_match": 0.0495,
+      "eval_f1": 0.4469,
+      "eval_gen_len": 13.6071,
+      "eval_loss": 1.1873786449432373,
+      "eval_precisions_1": 53.2071,
+      "eval_precisions_2": 26.5497,
+      "eval_precisions_3": 16.9576,
+      "eval_precisions_4": 11.5009,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.4559,
+      "eval_rouge2": 0.2639,
+      "eval_rougeL": 0.4417,
+      "eval_rougeLsum": 0.4413,
+      "eval_runtime": 3873.6961,
+      "eval_samples_per_second": 0.569,
+      "eval_steps_per_second": 0.569,
+      "eval_sys_len": 18272,
+      "eval_totals_1": 18272,
+      "eval_totals_2": 16068,
+      "eval_totals_3": 13864,
+      "eval_totals_4": 11660,
+      "step": 691
+    },
+    {
+      "epoch": 19.79,
+      "learning_rate": 0.0001,
+      "loss": 0.657,
+      "step": 720
+    },
+    {
+      "epoch": 19.79,
+      "eval_bleu": 19.8248,
+      "eval_bp": 0.8822,
+      "eval_counts_1": 9920,
+      "eval_counts_2": 4361,
+      "eval_counts_3": 2402,
+      "eval_counts_4": 1373,
+      "eval_exact_match": 0.0467,
+      "eval_f1": 0.4508,
+      "eval_gen_len": 14.2001,
+      "eval_loss": 1.1845453977584839,
+      "eval_precisions_1": 52.5312,
+      "eval_precisions_2": 26.1451,
+      "eval_precisions_3": 16.593,
+      "eval_precisions_4": 11.1881,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.4594,
+      "eval_rouge2": 0.2647,
+      "eval_rougeL": 0.4423,
+      "eval_rougeLsum": 0.4421,
+      "eval_runtime": 3958.6673,
+      "eval_samples_per_second": 0.557,
+      "eval_steps_per_second": 0.557,
+      "eval_sys_len": 18884,
+      "eval_totals_1": 18884,
+      "eval_totals_2": 16680,
+      "eval_totals_3": 14476,
+      "eval_totals_4": 12272,
+      "step": 720
+    },
+    {
+      "epoch": 19.79,
+      "step": 720,
+      "total_flos": 1.102412878184448e+18,
+      "train_loss": 1.3412119759453667,
+      "train_runtime": 128063.7495,
+      "train_samples_per_second": 1.455,
+      "train_steps_per_second": 0.006
+    }
+  ],
+  "logging_steps": 500,
+  "max_steps": 720,
+  "num_train_epochs": 20,
+  "save_steps": 500,
+  "total_flos": 1.102412878184448e+18,
+  "trial_name": null,
+  "trial_params": null
+}