Initial commit

.gitattributes

@@ -0,0 +1,4 @@
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
+runs/** filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,129 @@
+---
+language:
+- de
+tags:
+- question-generation
+- german
+- text2text-generation
+- generated_from_trainer
+datasets:
+- lmqg/qg_dequad
+metrics:
+- bleu4
+- f1
+- rouge
+- exact_match
+model-index:
+- name: german-jeopardy-mt5-base
+  results:
+  - task:
+      name: Sequence-to-sequence Language Modeling
+      type: text2text-generation
+    dataset:
+      name: lmqg/qg_dequad
+      type: default
+      args: default
+    metrics:
+    - name: BLEU-4
+      type: bleu4
+      value: 14.56
+    - name: F1
+      type: f1
+      value: 39.53
+    - name: ROUGE-1
+      type: rouge1
+      value: 40.62
+    - name: ROUGE-2
+      type: rouge2
+      value: 21.49
+    - name: ROUGE-L
+      type: rougel
+      value: 39.14
+    - name: ROUGE-Lsum
+      type: rougelsum
+      value: 39.13
+    - name: Exact Match
+      type: exact_match
+      value: 2.72
+---
+
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# german-jeopardy-mt5-base
+
+This model is a fine-tuned version of [google/mt5-base](https://huggingface.co/google/mt5-base) on the [lmqg/qg_dequad](https://huggingface.co/datasets/lmqg/qg_dequad) dataset.
+It achieves the following results on the evaluation set:
+- Loss: 1.66
+- Brevity Penalty: 0.9025
+- System Length: 18860
+- Reference Length: 20793
+- ROUGE-1: 40.62
+- ROUGE-2: 21.49
+- ROUGE-L: 39.14
+- ROUGE-Lsum: 39.13
+- Exact Match: 2.72
+- BLEU: 14.56
+- F1: 39.53
+
+## Model description
+
+See [google/mt5-base](https://huggingface.co/google/mt5-base) for the model architecture.  
+The model was trained on a single NVIDIA RTX 3090 GPU with 24GB of VRAM.
+
+## Intended uses & limitations
+
+This model can be used for question generation on German text.
+
+## Training and evaluation data
+
+See [lmqg/qg_dequad](https://huggingface.co/datasets/lmqg/qg_dequad).
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0001
+- train_batch_size: 4
+- eval_batch_size: 4
+- seed: 7
+- gradient_accumulation_steps: 16
+- total_train_batch_size: 64
+- optimizer: Adafactor
+- lr_scheduler_type: constant
+- num_epochs: 20
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Counts 1 | Counts 2 | Counts 3 | Counts 4 | Totals 1 | Totals 2 | Totals 3 | Totals 4 | Precisions 1 | Precisions 2 | Precisions 3 | Precisions 4 | Brevity Penalty | System Length | Reference Length | ROUGE-1 | ROUGE-2 | ROUGE-L | ROUGE-Lsum | Exact Match |  BLEU   | Mean Generated Length |   F1   |
+|:-------------:|:-----:|:----:|:---------------:|:--------:|:--------:|:--------:|:--------:|:--------:|:--------:|:--------:|:--------:|:------------:|:------------:|:------------:|:------------:|:---------------:|:-------------:|:----------------:|:-------:|:-------:|:-------:|:----------:|:-----------:|:-------:|:---------------------:|:------:|
+|    5.5131     |  1.0  | 145  |     1.8698      |   6032   |   1668   |   626    |   216    |  16023   |  13819   |  11615   |   9411   |   37.6459    |   12.0703    |    5.3896    |    2.2952    |     0.7216      |     16023     |      21250       | 0.2485  | 0.1011  | 0.2368  |   0.2366   |   0.0018    | 6.2485  |        12.6166        | 0.2406 |
+|    2.3946     |  2.0  | 291  |     1.5888      |   7325   |   2554   |   1178   |   558    |  16853   |  14649   |  12445   |  10241   |   43.4641    |   17.4346    |    9.4656    |    5.4487    |     0.7704      |     16853     |      21250       | 0.3226  | 0.1585  |  0.31   |    0.31    |   0.0145    | 10.8315 |        12.2582        | 0.3148 |
+|    2.0101     |  3.0  | 436  |     1.4997      |   7623   |   2764   |   1304   |   629    |  17042   |  14838   |  12634   |  10430   |   44.7307    |   18.6278    |   10.3214    |    6.0307    |     0.7812      |     17042     |      21250       | 0.3403  | 0.1723  | 0.3263  |   0.3263   |   0.0154    | 11.7891 |        12.6783        | 0.3315 |
+|    1.8073     |  4.0  | 582  |     1.4610      |   7728   |   2916   |   1415   |   707    |  16654   |  14450   |  12246   |  10042   |   46.4033    |   20.1799    |   11.5548    |    7.0404    |     0.7588      |     16654     |      21250       | 0.3461  | 0.1818  | 0.3324  |   0.3326   |   0.0168    | 12.6068 |        12.2963        | 0.3387 |
+|    1.6851     | 4.99  | 727  |     1.4357      |   7964   |   3059   |   1483   |   727    |  17381   |  15177   |  12973   |  10769   |   45.8201    |   20.1555    |   11.4314    |    6.7509    |     0.8004      |     17381     |      21250       | 0.3558  | 0.1888  | 0.3415  |   0.3414   |   0.0159    | 13.0784 |        12.7436        | 0.3483 |
+|    1.5642     |  6.0  | 873  |     1.4003      |   8299   |   3224   |   1592   |   788    |  17351   |  15147   |  12943   |  10739   |   47.8301    |   21.2847    |   12.3001    |    7.3377    |     0.7987      |     17351     |      21250       | 0.3814  | 0.2025  | 0.3684  |   0.3685   |   0.0204    | 13.9065 |        12.9569        | 0.3736 |
+|    1.4756     | 6.99  | 1018 |     1.3779      |   8640   |   3430   |   1712   |   879    |  17669   |  15465   |  13261   |  11057   |   48.8992    |   22.1791    |    12.91     |    7.9497    |     0.8165      |     17669     |      21250       | 0.3971  | 0.2133  | 0.3828  |   0.3826   |    0.025    | 14.9146 |        13.1084        | 0.3892 |
+|    1.3792     |  8.0  | 1164 |     1.3624      |   8732   |   3417   |   1712   |   871    |  17996   |  15792   |  13588   |  11384   |   48.5219    |   21.6375    |   12.5994    |    7.6511    |     0.8346      |     17996     |      21250       | 0.4003  | 0.2131  | 0.3852  |   0.3849   |   0.0245    | 14.8859 |        13.3748        | 0.3917 |
+|    1.3133     |  9.0  | 1310 |     1.3630      |   8804   |   3500   |   1754   |   920    |  17661   |  15457   |  13253   |  11049   |    49.85     |   22.6435    |   13.2347    |    8.3265    |     0.8161      |     17661     |      21250       | 0.4078  |  0.219  | 0.3932  |   0.3935   |    0.025    | 15.3264 |        13.2019        |  0.4   |
+|     1.261     | 10.0  | 1455 |     1.3685      |   8910   |   3602   |   1849   |   1000   |  17709   |  15505   |  13301   |  11097   |   50.3134    |   23.2312    |   13.9012    |    9.0114    |     0.8188      |     17709     |      21250       | 0.4135  |  0.223  | 0.3991  |   0.3992   |   0.0295    | 16.0163 |        13.1892        | 0.4055 |
+|    1.1897     | 11.0  | 1601 |     1.3639      |   9096   |   3690   |   1902   |   1012   |  18261   |  16057   |  13853   |  11649   |   49.8111    |   22.9806    |   13.7299    |    8.6874    |      0.849      |     18261     |      21250       | 0.4201  | 0.2289  | 0.4059  |   0.4057   |   0.0281    | 16.3202 |        13.5077        | 0.4121 |
+|    1.1453     | 11.99 | 1746 |     1.3610      |   9106   |   3735   |   1932   |   1023   |  18329   |  16125   |  13921   |  11717   |   49.6808    |   23.1628    |   13.8783    |    8.7309    |     0.8527      |     18329     |      21250       | 0.4173  | 0.2303  | 0.4026  |   0.4025   |   0.0281    | 16.4772 |        13.8013        | 0.4099 |
+|    1.0858     | 13.0  | 1892 |     1.3716      |   9245   |   3778   |   1955   |   1049   |  18556   |  16352   |  14148   |  11944   |   49.8222    |   23.1042    |   13.8182    |    8.7827    |     0.8649      |     18556     |      21250       | 0.4244  | 0.2327  |  0.409  |   0.409    |   0.0322    | 16.7204 |        13.8144        | 0.417  |
+|    1.0472     | 13.99 | 2037 |     1.3770      |   9166   |   3756   |   1946   |   1054   |  18315   |  16111   |  13907   |  11703   |   50.0464    |   23.3133    |    13.993    |    9.0062    |     0.8519      |     18315     |      21250       | 0.4216  | 0.2311  | 0.4068  |   0.4067   |   0.0309    | 16.6825 |        13.8099        | 0.4143 |
+|    0.9953     | 15.0  | 2183 |     1.3881      |   9342   |   3926   |   2046   |   1108   |  18132   |  15928   |  13724   |  11520   |   51.5222    |   24.6484    |   14.9082    |    9.6181    |      0.842      |     18132     |      21250       | 0.4328  | 0.2418  | 0.4171  |   0.4171   |   0.0327    | 17.3937 |        13.5023        | 0.4258 |
+|    0.9509     | 16.0  | 2329 |     1.4016      |   9330   |   3894   |   2024   |   1084   |  18672   |  16468   |  14264   |  12060   |   49.9679    |   23.6459    |   14.1896    |    8.9884    |      0.871      |     18672     |      21250       | 0.4269  |  0.237  | 0.4123  |   0.4122   |   0.0313    | 17.1618 |        13.956         | 0.4198 |
+|    0.9183     | 17.0  | 2474 |     1.4152      |   9303   |   3824   |   1979   |   1084   |  18476   |  16272   |  14068   |  11864   |   50.3518    |   23.5005    |   14.0674    |    9.1369    |     0.8606      |     18476     |      21250       | 0.4269  | 0.2345  | 0.4121  |   0.4122   |   0.0327    | 16.995  |        13.7854        | 0.4199 |
+|    0.8696     | 18.0  | 2620 |     1.4404      |   9184   |   3798   |   1993   |   1085   |  18379   |  16175   |  13971   |  11767   |   49.9701    |   23.4807    |   14.2653    |    9.2207    |     0.8554      |     18379     |      21250       | 0.4218  | 0.2333  | 0.4076  |   0.4074   |    0.034    | 16.9541 |        13.726         | 0.4148 |
+|    0.8389     | 19.0  | 2765 |     1.4360      |   9476   |   4000   |   2092   |   1139   |  19003   |  16799   |  14595   |  12391   |   49.8658    |   23.8109    |   14.3337    |    9.1922    |     0.8885      |     19003     |      21250       | 0.4307  | 0.2406  | 0.4161  |   0.416    |   0.0299    |  17.67  |        14.2064        | 0.4239 |
+|    0.7993     | 19.92 | 2900 |     1.4545      |   9464   |   3970   |   2078   |   1126   |  18741   |  16537   |  14333   |  12129   |   50.4989    |   24.0068    |    14.498    |    9.2835    |     0.8747      |     18741     |      21250       | 0.4349  | 0.2424  | 0.4194  |   0.4192   |   0.0327    | 17.5799 |        13.9959        | 0.4269 |
+
+
+### Framework versions
+
+- Transformers 4.32.1
+- Pytorch 2.1.0
+- Datasets 2.12.0
+- Tokenizers 0.13.3

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<hl>": 250100
+}

all_results.json

@@ -0,0 +1,64 @@
+{
+    "epoch": 19.92,
+    "eval_bleu": 17.5799,
+    "eval_bp": 0.8747,
+    "eval_counts_1": 9464,
+    "eval_counts_2": 3970,
+    "eval_counts_3": 2078,
+    "eval_counts_4": 1126,
+    "eval_exact_match": 0.0327,
+    "eval_f1": 0.4269,
+    "eval_gen_len": 13.9959,
+    "eval_loss": 1.454466700553894,
+    "eval_precisions_1": 50.4989,
+    "eval_precisions_2": 24.0068,
+    "eval_precisions_3": 14.498,
+    "eval_precisions_4": 9.2835,
+    "eval_ref_len": 21250,
+    "eval_rouge1": 0.4349,
+    "eval_rouge2": 0.2424,
+    "eval_rougeL": 0.4194,
+    "eval_rougeLsum": 0.4192,
+    "eval_runtime": 476.6122,
+    "eval_samples": 2204,
+    "eval_samples_per_second": 4.624,
+    "eval_steps_per_second": 1.156,
+    "eval_sys_len": 18741,
+    "eval_totals_1": 18741,
+    "eval_totals_2": 16537,
+    "eval_totals_3": 14333,
+    "eval_totals_4": 12129,
+    "predict_bleu": 14.049,
+    "predict_bp": 0.9007,
+    "predict_counts_1": 8655,
+    "predict_counts_2": 3255,
+    "predict_counts_3": 1539,
+    "predict_counts_4": 752,
+    "predict_exact_match": 0.0254,
+    "predict_f1": 0.3846,
+    "predict_gen_len": 14.2414,
+    "predict_loss": 1.751596212387085,
+    "predict_precisions_1": 45.9761,
+    "predict_precisions_2": 19.5837,
+    "predict_precisions_3": 10.6749,
+    "predict_precisions_4": 6.1574,
+    "predict_ref_len": 20793,
+    "predict_rouge1": 0.3957,
+    "predict_rouge2": 0.2081,
+    "predict_rougeL": 0.3818,
+    "predict_rougeLsum": 0.3817,
+    "predict_runtime": 482.487,
+    "predict_samples": 2204,
+    "predict_samples_per_second": 4.568,
+    "predict_steps_per_second": 1.142,
+    "predict_sys_len": 18825,
+    "predict_totals_1": 18825,
+    "predict_totals_2": 16621,
+    "predict_totals_3": 14417,
+    "predict_totals_4": 12213,
+    "train_loss": 1.5141178552035628,
+    "train_runtime": 27637.7455,
+    "train_samples": 9314,
+    "train_samples_per_second": 6.74,
+    "train_steps_per_second": 0.105
+}

config.json

@@ -0,0 +1,36 @@
+{
+  "_name_or_path": "output/models/german-jeopardy-mt5-base-1k-64-constant",
+  "architectures": [
+    "MT5ForConditionalGeneration"
+  ],
+  "classifier_dropout": 0.0,
+  "d_ff": 2048,
+  "d_kv": 64,
+  "d_model": 768,
+  "decoder_start_token_id": 0,
+  "dense_act_fn": "gelu_new",
+  "dropout_rate": 0.1,
+  "eos_token_id": 1,
+  "feed_forward_proj": "gated-gelu",
+  "initializer_factor": 1.0,
+  "is_encoder_decoder": true,
+  "is_gated_act": true,
+  "layer_norm_epsilon": 1e-06,
+  "length_penalty": 0.0,
+  "max_length": 64,
+  "model_type": "mt5",
+  "num_beams": 4,
+  "num_decoder_layers": 12,
+  "num_heads": 12,
+  "num_layers": 12,
+  "output_past": true,
+  "pad_token_id": 0,
+  "relative_attention_max_distance": 128,
+  "relative_attention_num_buckets": 32,
+  "tie_word_embeddings": false,
+  "tokenizer_class": "T5Tokenizer",
+  "torch_dtype": "float32",
+  "transformers_version": "4.32.1",
+  "use_cache": true,
+  "vocab_size": 250112
+}

generation_config.json

@@ -0,0 +1,9 @@
+{
+  "decoder_start_token_id": 0,
+  "eos_token_id": 1,
+  "length_penalty": 0.0,
+  "max_length": 64,
+  "num_beams": 4,
+  "pad_token_id": 0,
+  "transformers_version": "4.32.1"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32ec8d6ff7a2f7e8e25cbd40c511a16ff3e31943211925e77443a049ec6a8fb9
+size 2329638768

runs/Oct19_15-31-11_MARVIN-PC/events.out.tfevents.1697722302.MARVIN-PC.21928.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:733db217c37a7d793c1effe1724bc46d7abceda17ca216ae76f2e10b234d28a7
+size 37303

runs/Oct19_15-31-11_MARVIN-PC/events.out.tfevents.1697750423.MARVIN-PC.21928.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b142942ee3bc070200e7be88b7e9dab9a483d71318d00c5d18e09cc8ee366f5
+size 1550

special_tokens_map.json

@@ -0,0 +1,8 @@
+{
+  "additional_special_tokens": [
+    "<hl>"
+  ],
+  "eos_token": "</s>",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
+}

spiece.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef78f86560d809067d12bac6c09f19a462cb3af3f54d2b8acbba26e1433125d6
+size 4309802

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bca227cebb484387622222f8a48126e730090bd37f37ec0e450947cd52f5477f
+size 16315565

tokenizer_config.json

@@ -0,0 +1,19 @@
+{
+  "additional_special_tokens": null,
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "</s>",
+  "extra_ids": 0,
+  "legacy": true,
+  "max_length": 64,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_to_multiple_of": null,
+  "pad_token": "<pad>",
+  "pad_token_type_id": 0,
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "stride": 0,
+  "tokenizer_class": "T5Tokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
+  "unk_token": "<unk>"
+}

trainer_state.json

@@ -0,0 +1,768 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 19.922713610991842,
+  "eval_steps": 500,
+  "global_step": 2900,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 1.0,
+      "learning_rate": 0.0001,
+      "loss": 5.5131,
+      "step": 145
+    },
+    {
+      "epoch": 1.0,
+      "eval_bleu": 6.2485,
+      "eval_bp": 0.7216,
+      "eval_counts_1": 6032,
+      "eval_counts_2": 1668,
+      "eval_counts_3": 626,
+      "eval_counts_4": 216,
+      "eval_exact_match": 0.0018,
+      "eval_f1": 0.2406,
+      "eval_gen_len": 12.6166,
+      "eval_loss": 1.8697563409805298,
+      "eval_precisions_1": 37.6459,
+      "eval_precisions_2": 12.0703,
+      "eval_precisions_3": 5.3896,
+      "eval_precisions_4": 2.2952,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.2485,
+      "eval_rouge2": 0.1011,
+      "eval_rougeL": 0.2368,
+      "eval_rougeLsum": 0.2366,
+      "eval_runtime": 467.1177,
+      "eval_samples_per_second": 4.718,
+      "eval_steps_per_second": 1.18,
+      "eval_sys_len": 16023,
+      "eval_totals_1": 16023,
+      "eval_totals_2": 13819,
+      "eval_totals_3": 11615,
+      "eval_totals_4": 9411,
+      "step": 145
+    },
+    {
+      "epoch": 2.0,
+      "learning_rate": 0.0001,
+      "loss": 2.3946,
+      "step": 291
+    },
+    {
+      "epoch": 2.0,
+      "eval_bleu": 10.8315,
+      "eval_bp": 0.7704,
+      "eval_counts_1": 7325,
+      "eval_counts_2": 2554,
+      "eval_counts_3": 1178,
+      "eval_counts_4": 558,
+      "eval_exact_match": 0.0145,
+      "eval_f1": 0.3148,
+      "eval_gen_len": 12.2582,
+      "eval_loss": 1.58878493309021,
+      "eval_precisions_1": 43.4641,
+      "eval_precisions_2": 17.4346,
+      "eval_precisions_3": 9.4656,
+      "eval_precisions_4": 5.4487,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.3226,
+      "eval_rouge2": 0.1585,
+      "eval_rougeL": 0.31,
+      "eval_rougeLsum": 0.31,
+      "eval_runtime": 528.6481,
+      "eval_samples_per_second": 4.169,
+      "eval_steps_per_second": 1.042,
+      "eval_sys_len": 16853,
+      "eval_totals_1": 16853,
+      "eval_totals_2": 14649,
+      "eval_totals_3": 12445,
+      "eval_totals_4": 10241,
+      "step": 291
+    },
+    {
+      "epoch": 3.0,
+      "learning_rate": 0.0001,
+      "loss": 2.0101,
+      "step": 436
+    },
+    {
+      "epoch": 3.0,
+      "eval_bleu": 11.7891,
+      "eval_bp": 0.7812,
+      "eval_counts_1": 7623,
+      "eval_counts_2": 2764,
+      "eval_counts_3": 1304,
+      "eval_counts_4": 629,
+      "eval_exact_match": 0.0154,
+      "eval_f1": 0.3315,
+      "eval_gen_len": 12.6783,
+      "eval_loss": 1.4997321367263794,
+      "eval_precisions_1": 44.7307,
+      "eval_precisions_2": 18.6278,
+      "eval_precisions_3": 10.3214,
+      "eval_precisions_4": 6.0307,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.3403,
+      "eval_rouge2": 0.1723,
+      "eval_rougeL": 0.3263,
+      "eval_rougeLsum": 0.3263,
+      "eval_runtime": 451.1882,
+      "eval_samples_per_second": 4.885,
+      "eval_steps_per_second": 1.221,
+      "eval_sys_len": 17042,
+      "eval_totals_1": 17042,
+      "eval_totals_2": 14838,
+      "eval_totals_3": 12634,
+      "eval_totals_4": 10430,
+      "step": 436
+    },
+    {
+      "epoch": 4.0,
+      "learning_rate": 0.0001,
+      "loss": 1.8073,
+      "step": 582
+    },
+    {
+      "epoch": 4.0,
+      "eval_bleu": 12.6068,
+      "eval_bp": 0.7588,
+      "eval_counts_1": 7728,
+      "eval_counts_2": 2916,
+      "eval_counts_3": 1415,
+      "eval_counts_4": 707,
+      "eval_exact_match": 0.0168,
+      "eval_f1": 0.3387,
+      "eval_gen_len": 12.2963,
+      "eval_loss": 1.4610050916671753,
+      "eval_precisions_1": 46.4033,
+      "eval_precisions_2": 20.1799,
+      "eval_precisions_3": 11.5548,
+      "eval_precisions_4": 7.0404,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.3461,
+      "eval_rouge2": 0.1818,
+      "eval_rougeL": 0.3324,
+      "eval_rougeLsum": 0.3326,
+      "eval_runtime": 433.3953,
+      "eval_samples_per_second": 5.085,
+      "eval_steps_per_second": 1.271,
+      "eval_sys_len": 16654,
+      "eval_totals_1": 16654,
+      "eval_totals_2": 14450,
+      "eval_totals_3": 12246,
+      "eval_totals_4": 10042,
+      "step": 582
+    },
+    {
+      "epoch": 4.99,
+      "learning_rate": 0.0001,
+      "loss": 1.6851,
+      "step": 727
+    },
+    {
+      "epoch": 4.99,
+      "eval_bleu": 13.0784,
+      "eval_bp": 0.8004,
+      "eval_counts_1": 7964,
+      "eval_counts_2": 3059,
+      "eval_counts_3": 1483,
+      "eval_counts_4": 727,
+      "eval_exact_match": 0.0159,
+      "eval_f1": 0.3483,
+      "eval_gen_len": 12.7436,
+      "eval_loss": 1.4356882572174072,
+      "eval_precisions_1": 45.8201,
+      "eval_precisions_2": 20.1555,
+      "eval_precisions_3": 11.4314,
+      "eval_precisions_4": 6.7509,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.3558,
+      "eval_rouge2": 0.1888,
+      "eval_rougeL": 0.3415,
+      "eval_rougeLsum": 0.3414,
+      "eval_runtime": 452.1483,
+      "eval_samples_per_second": 4.875,
+      "eval_steps_per_second": 1.219,
+      "eval_sys_len": 17381,
+      "eval_totals_1": 17381,
+      "eval_totals_2": 15177,
+      "eval_totals_3": 12973,
+      "eval_totals_4": 10769,
+      "step": 727
+    },
+    {
+      "epoch": 6.0,
+      "learning_rate": 0.0001,
+      "loss": 1.5642,
+      "step": 873
+    },
+    {
+      "epoch": 6.0,
+      "eval_bleu": 13.9065,
+      "eval_bp": 0.7987,
+      "eval_counts_1": 8299,
+      "eval_counts_2": 3224,
+      "eval_counts_3": 1592,
+      "eval_counts_4": 788,
+      "eval_exact_match": 0.0204,
+      "eval_f1": 0.3736,
+      "eval_gen_len": 12.9569,
+      "eval_loss": 1.4003357887268066,
+      "eval_precisions_1": 47.8301,
+      "eval_precisions_2": 21.2847,
+      "eval_precisions_3": 12.3001,
+      "eval_precisions_4": 7.3377,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.3814,
+      "eval_rouge2": 0.2025,
+      "eval_rougeL": 0.3684,
+      "eval_rougeLsum": 0.3685,
+      "eval_runtime": 450.2054,
+      "eval_samples_per_second": 4.896,
+      "eval_steps_per_second": 1.224,
+      "eval_sys_len": 17351,
+      "eval_totals_1": 17351,
+      "eval_totals_2": 15147,
+      "eval_totals_3": 12943,
+      "eval_totals_4": 10739,
+      "step": 873
+    },
+    {
+      "epoch": 6.99,
+      "learning_rate": 0.0001,
+      "loss": 1.4756,
+      "step": 1018
+    },
+    {
+      "epoch": 6.99,
+      "eval_bleu": 14.9146,
+      "eval_bp": 0.8165,
+      "eval_counts_1": 8640,
+      "eval_counts_2": 3430,
+      "eval_counts_3": 1712,
+      "eval_counts_4": 879,
+      "eval_exact_match": 0.025,
+      "eval_f1": 0.3892,
+      "eval_gen_len": 13.1084,
+      "eval_loss": 1.3778630495071411,
+      "eval_precisions_1": 48.8992,
+      "eval_precisions_2": 22.1791,
+      "eval_precisions_3": 12.91,
+      "eval_precisions_4": 7.9497,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.3971,
+      "eval_rouge2": 0.2133,
+      "eval_rougeL": 0.3828,
+      "eval_rougeLsum": 0.3826,
+      "eval_runtime": 753.2935,
+      "eval_samples_per_second": 2.926,
+      "eval_steps_per_second": 0.731,
+      "eval_sys_len": 17669,
+      "eval_totals_1": 17669,
+      "eval_totals_2": 15465,
+      "eval_totals_3": 13261,
+      "eval_totals_4": 11057,
+      "step": 1018
+    },
+    {
+      "epoch": 8.0,
+      "learning_rate": 0.0001,
+      "loss": 1.3792,
+      "step": 1164
+    },
+    {
+      "epoch": 8.0,
+      "eval_bleu": 14.8859,
+      "eval_bp": 0.8346,
+      "eval_counts_1": 8732,
+      "eval_counts_2": 3417,
+      "eval_counts_3": 1712,
+      "eval_counts_4": 871,
+      "eval_exact_match": 0.0245,
+      "eval_f1": 0.3917,
+      "eval_gen_len": 13.3748,
+      "eval_loss": 1.362410306930542,
+      "eval_precisions_1": 48.5219,
+      "eval_precisions_2": 21.6375,
+      "eval_precisions_3": 12.5994,
+      "eval_precisions_4": 7.6511,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.4003,
+      "eval_rouge2": 0.2131,
+      "eval_rougeL": 0.3852,
+      "eval_rougeLsum": 0.3849,
+      "eval_runtime": 699.0977,
+      "eval_samples_per_second": 3.153,
+      "eval_steps_per_second": 0.788,
+      "eval_sys_len": 17996,
+      "eval_totals_1": 17996,
+      "eval_totals_2": 15792,
+      "eval_totals_3": 13588,
+      "eval_totals_4": 11384,
+      "step": 1164
+    },
+    {
+      "epoch": 9.0,
+      "learning_rate": 0.0001,
+      "loss": 1.3133,
+      "step": 1310
+    },
+    {
+      "epoch": 9.0,
+      "eval_bleu": 15.3264,
+      "eval_bp": 0.8161,
+      "eval_counts_1": 8804,
+      "eval_counts_2": 3500,
+      "eval_counts_3": 1754,
+      "eval_counts_4": 920,
+      "eval_exact_match": 0.025,
+      "eval_f1": 0.4,
+      "eval_gen_len": 13.2019,
+      "eval_loss": 1.3630096912384033,
+      "eval_precisions_1": 49.85,
+      "eval_precisions_2": 22.6435,
+      "eval_precisions_3": 13.2347,
+      "eval_precisions_4": 8.3265,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.4078,
+      "eval_rouge2": 0.219,
+      "eval_rougeL": 0.3932,
+      "eval_rougeLsum": 0.3935,
+      "eval_runtime": 465.2887,
+      "eval_samples_per_second": 4.737,
+      "eval_steps_per_second": 1.184,
+      "eval_sys_len": 17661,
+      "eval_totals_1": 17661,
+      "eval_totals_2": 15457,
+      "eval_totals_3": 13253,
+      "eval_totals_4": 11049,
+      "step": 1310
+    },
+    {
+      "epoch": 10.0,
+      "learning_rate": 0.0001,
+      "loss": 1.261,
+      "step": 1455
+    },
+    {
+      "epoch": 10.0,
+      "eval_bleu": 16.0163,
+      "eval_bp": 0.8188,
+      "eval_counts_1": 8910,
+      "eval_counts_2": 3602,
+      "eval_counts_3": 1849,
+      "eval_counts_4": 1000,
+      "eval_exact_match": 0.0295,
+      "eval_f1": 0.4055,
+      "eval_gen_len": 13.1892,
+      "eval_loss": 1.3685479164123535,
+      "eval_precisions_1": 50.3134,
+      "eval_precisions_2": 23.2312,
+      "eval_precisions_3": 13.9012,
+      "eval_precisions_4": 9.0114,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.4135,
+      "eval_rouge2": 0.223,
+      "eval_rougeL": 0.3991,
+      "eval_rougeLsum": 0.3992,
+      "eval_runtime": 491.3102,
+      "eval_samples_per_second": 4.486,
+      "eval_steps_per_second": 1.121,
+      "eval_sys_len": 17709,
+      "eval_totals_1": 17709,
+      "eval_totals_2": 15505,
+      "eval_totals_3": 13301,
+      "eval_totals_4": 11097,
+      "step": 1455
+    },
+    {
+      "epoch": 11.0,
+      "learning_rate": 0.0001,
+      "loss": 1.1897,
+      "step": 1601
+    },
+    {
+      "epoch": 11.0,
+      "eval_bleu": 16.3202,
+      "eval_bp": 0.849,
+      "eval_counts_1": 9096,
+      "eval_counts_2": 3690,
+      "eval_counts_3": 1902,
+      "eval_counts_4": 1012,
+      "eval_exact_match": 0.0281,
+      "eval_f1": 0.4121,
+      "eval_gen_len": 13.5077,
+      "eval_loss": 1.3638867139816284,
+      "eval_precisions_1": 49.8111,
+      "eval_precisions_2": 22.9806,
+      "eval_precisions_3": 13.7299,
+      "eval_precisions_4": 8.6874,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.4201,
+      "eval_rouge2": 0.2289,
+      "eval_rougeL": 0.4059,
+      "eval_rougeLsum": 0.4057,
+      "eval_runtime": 536.9399,
+      "eval_samples_per_second": 4.105,
+      "eval_steps_per_second": 1.026,
+      "eval_sys_len": 18261,
+      "eval_totals_1": 18261,
+      "eval_totals_2": 16057,
+      "eval_totals_3": 13853,
+      "eval_totals_4": 11649,
+      "step": 1601
+    },
+    {
+      "epoch": 11.99,
+      "learning_rate": 0.0001,
+      "loss": 1.1453,
+      "step": 1746
+    },
+    {
+      "epoch": 11.99,
+      "eval_bleu": 16.4772,
+      "eval_bp": 0.8527,
+      "eval_counts_1": 9106,
+      "eval_counts_2": 3735,
+      "eval_counts_3": 1932,
+      "eval_counts_4": 1023,
+      "eval_exact_match": 0.0281,
+      "eval_f1": 0.4099,
+      "eval_gen_len": 13.8013,
+      "eval_loss": 1.3609519004821777,
+      "eval_precisions_1": 49.6808,
+      "eval_precisions_2": 23.1628,
+      "eval_precisions_3": 13.8783,
+      "eval_precisions_4": 8.7309,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.4173,
+      "eval_rouge2": 0.2303,
+      "eval_rougeL": 0.4026,
+      "eval_rougeLsum": 0.4025,
+      "eval_runtime": 617.7899,
+      "eval_samples_per_second": 3.568,
+      "eval_steps_per_second": 0.892,
+      "eval_sys_len": 18329,
+      "eval_totals_1": 18329,
+      "eval_totals_2": 16125,
+      "eval_totals_3": 13921,
+      "eval_totals_4": 11717,
+      "step": 1746
+    },
+    {
+      "epoch": 13.0,
+      "learning_rate": 0.0001,
+      "loss": 1.0858,
+      "step": 1892
+    },
+    {
+      "epoch": 13.0,
+      "eval_bleu": 16.7204,
+      "eval_bp": 0.8649,
+      "eval_counts_1": 9245,
+      "eval_counts_2": 3778,
+      "eval_counts_3": 1955,
+      "eval_counts_4": 1049,
+      "eval_exact_match": 0.0322,
+      "eval_f1": 0.417,
+      "eval_gen_len": 13.8144,
+      "eval_loss": 1.3716095685958862,
+      "eval_precisions_1": 49.8222,
+      "eval_precisions_2": 23.1042,
+      "eval_precisions_3": 13.8182,
+      "eval_precisions_4": 8.7827,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.4244,
+      "eval_rouge2": 0.2327,
+      "eval_rougeL": 0.409,
+      "eval_rougeLsum": 0.409,
+      "eval_runtime": 504.2774,
+      "eval_samples_per_second": 4.371,
+      "eval_steps_per_second": 1.093,
+      "eval_sys_len": 18556,
+      "eval_totals_1": 18556,
+      "eval_totals_2": 16352,
+      "eval_totals_3": 14148,
+      "eval_totals_4": 11944,
+      "step": 1892
+    },
+    {
+      "epoch": 13.99,
+      "learning_rate": 0.0001,
+      "loss": 1.0472,
+      "step": 2037
+    },
+    {
+      "epoch": 13.99,
+      "eval_bleu": 16.6825,
+      "eval_bp": 0.8519,
+      "eval_counts_1": 9166,
+      "eval_counts_2": 3756,
+      "eval_counts_3": 1946,
+      "eval_counts_4": 1054,
+      "eval_exact_match": 0.0309,
+      "eval_f1": 0.4143,
+      "eval_gen_len": 13.8099,
+      "eval_loss": 1.3770091533660889,
+      "eval_precisions_1": 50.0464,
+      "eval_precisions_2": 23.3133,
+      "eval_precisions_3": 13.993,
+      "eval_precisions_4": 9.0062,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.4216,
+      "eval_rouge2": 0.2311,
+      "eval_rougeL": 0.4068,
+      "eval_rougeLsum": 0.4067,
+      "eval_runtime": 581.2707,
+      "eval_samples_per_second": 3.792,
+      "eval_steps_per_second": 0.948,
+      "eval_sys_len": 18315,
+      "eval_totals_1": 18315,
+      "eval_totals_2": 16111,
+      "eval_totals_3": 13907,
+      "eval_totals_4": 11703,
+      "step": 2037
+    },
+    {
+      "epoch": 15.0,
+      "learning_rate": 0.0001,
+      "loss": 0.9953,
+      "step": 2183
+    },
+    {
+      "epoch": 15.0,
+      "eval_bleu": 17.3937,
+      "eval_bp": 0.842,
+      "eval_counts_1": 9342,
+      "eval_counts_2": 3926,
+      "eval_counts_3": 2046,
+      "eval_counts_4": 1108,
+      "eval_exact_match": 0.0327,
+      "eval_f1": 0.4258,
+      "eval_gen_len": 13.5023,
+      "eval_loss": 1.3880597352981567,
+      "eval_precisions_1": 51.5222,
+      "eval_precisions_2": 24.6484,
+      "eval_precisions_3": 14.9082,
+      "eval_precisions_4": 9.6181,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.4328,
+      "eval_rouge2": 0.2418,
+      "eval_rougeL": 0.4171,
+      "eval_rougeLsum": 0.4171,
+      "eval_runtime": 718.2329,
+      "eval_samples_per_second": 3.069,
+      "eval_steps_per_second": 0.767,
+      "eval_sys_len": 18132,
+      "eval_totals_1": 18132,
+      "eval_totals_2": 15928,
+      "eval_totals_3": 13724,
+      "eval_totals_4": 11520,
+      "step": 2183
+    },
+    {
+      "epoch": 16.0,
+      "learning_rate": 0.0001,
+      "loss": 0.9509,
+      "step": 2329
+    },
+    {
+      "epoch": 16.0,
+      "eval_bleu": 17.1618,
+      "eval_bp": 0.871,
+      "eval_counts_1": 9330,
+      "eval_counts_2": 3894,
+      "eval_counts_3": 2024,
+      "eval_counts_4": 1084,
+      "eval_exact_match": 0.0313,
+      "eval_f1": 0.4198,
+      "eval_gen_len": 13.956,
+      "eval_loss": 1.401639461517334,
+      "eval_precisions_1": 49.9679,
+      "eval_precisions_2": 23.6459,
+      "eval_precisions_3": 14.1896,
+      "eval_precisions_4": 8.9884,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.4269,
+      "eval_rouge2": 0.237,
+      "eval_rougeL": 0.4123,
+      "eval_rougeLsum": 0.4122,
+      "eval_runtime": 632.3222,
+      "eval_samples_per_second": 3.486,
+      "eval_steps_per_second": 0.871,
+      "eval_sys_len": 18672,
+      "eval_totals_1": 18672,
+      "eval_totals_2": 16468,
+      "eval_totals_3": 14264,
+      "eval_totals_4": 12060,
+      "step": 2329
+    },
+    {
+      "epoch": 17.0,
+      "learning_rate": 0.0001,
+      "loss": 0.9183,
+      "step": 2474
+    },
+    {
+      "epoch": 17.0,
+      "eval_bleu": 16.995,
+      "eval_bp": 0.8606,
+      "eval_counts_1": 9303,
+      "eval_counts_2": 3824,
+      "eval_counts_3": 1979,
+      "eval_counts_4": 1084,
+      "eval_exact_match": 0.0327,
+      "eval_f1": 0.4199,
+      "eval_gen_len": 13.7854,
+      "eval_loss": 1.4152026176452637,
+      "eval_precisions_1": 50.3518,
+      "eval_precisions_2": 23.5005,
+      "eval_precisions_3": 14.0674,
+      "eval_precisions_4": 9.1369,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.4269,
+      "eval_rouge2": 0.2345,
+      "eval_rougeL": 0.4121,
+      "eval_rougeLsum": 0.4122,
+      "eval_runtime": 466.5423,
+      "eval_samples_per_second": 4.724,
+      "eval_steps_per_second": 1.181,
+      "eval_sys_len": 18476,
+      "eval_totals_1": 18476,
+      "eval_totals_2": 16272,
+      "eval_totals_3": 14068,
+      "eval_totals_4": 11864,
+      "step": 2474
+    },
+    {
+      "epoch": 18.0,
+      "learning_rate": 0.0001,
+      "loss": 0.8696,
+      "step": 2620
+    },
+    {
+      "epoch": 18.0,
+      "eval_bleu": 16.9541,
+      "eval_bp": 0.8554,
+      "eval_counts_1": 9184,
+      "eval_counts_2": 3798,
+      "eval_counts_3": 1993,
+      "eval_counts_4": 1085,
+      "eval_exact_match": 0.034,
+      "eval_f1": 0.4148,
+      "eval_gen_len": 13.726,
+      "eval_loss": 1.44040048122406,
+      "eval_precisions_1": 49.9701,
+      "eval_precisions_2": 23.4807,
+      "eval_precisions_3": 14.2653,
+      "eval_precisions_4": 9.2207,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.4218,
+      "eval_rouge2": 0.2333,
+      "eval_rougeL": 0.4076,
+      "eval_rougeLsum": 0.4074,
+      "eval_runtime": 470.6343,
+      "eval_samples_per_second": 4.683,
+      "eval_steps_per_second": 1.171,
+      "eval_sys_len": 18379,
+      "eval_totals_1": 18379,
+      "eval_totals_2": 16175,
+      "eval_totals_3": 13971,
+      "eval_totals_4": 11767,
+      "step": 2620
+    },
+    {
+      "epoch": 19.0,
+      "learning_rate": 0.0001,
+      "loss": 0.8389,
+      "step": 2765
+    },
+    {
+      "epoch": 19.0,
+      "eval_bleu": 17.67,
+      "eval_bp": 0.8885,
+      "eval_counts_1": 9476,
+      "eval_counts_2": 4000,
+      "eval_counts_3": 2092,
+      "eval_counts_4": 1139,
+      "eval_exact_match": 0.0299,
+      "eval_f1": 0.4239,
+      "eval_gen_len": 14.2064,
+      "eval_loss": 1.4360300302505493,
+      "eval_precisions_1": 49.8658,
+      "eval_precisions_2": 23.8109,
+      "eval_precisions_3": 14.3337,
+      "eval_precisions_4": 9.1922,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.4307,
+      "eval_rouge2": 0.2406,
+      "eval_rougeL": 0.4161,
+      "eval_rougeLsum": 0.416,
+      "eval_runtime": 480.4816,
+      "eval_samples_per_second": 4.587,
+      "eval_steps_per_second": 1.147,
+      "eval_sys_len": 19003,
+      "eval_totals_1": 19003,
+      "eval_totals_2": 16799,
+      "eval_totals_3": 14595,
+      "eval_totals_4": 12391,
+      "step": 2765
+    },
+    {
+      "epoch": 19.92,
+      "learning_rate": 0.0001,
+      "loss": 0.7993,
+      "step": 2900
+    },
+    {
+      "epoch": 19.92,
+      "eval_bleu": 17.5799,
+      "eval_bp": 0.8747,
+      "eval_counts_1": 9464,
+      "eval_counts_2": 3970,
+      "eval_counts_3": 2078,
+      "eval_counts_4": 1126,
+      "eval_exact_match": 0.0327,
+      "eval_f1": 0.4269,
+      "eval_gen_len": 13.9959,
+      "eval_loss": 1.454466700553894,
+      "eval_precisions_1": 50.4989,
+      "eval_precisions_2": 24.0068,
+      "eval_precisions_3": 14.498,
+      "eval_precisions_4": 9.2835,
+      "eval_ref_len": 21250,
+      "eval_rouge1": 0.4349,
+      "eval_rouge2": 0.2424,
+      "eval_rougeL": 0.4194,
+      "eval_rougeLsum": 0.4192,
+      "eval_runtime": 476.8512,
+      "eval_samples_per_second": 4.622,
+      "eval_steps_per_second": 1.155,
+      "eval_sys_len": 18741,
+      "eval_totals_1": 18741,
+      "eval_totals_2": 16537,
+      "eval_totals_3": 14333,
+      "eval_totals_4": 12129,
+      "step": 2900
+    },
+    {
+      "epoch": 19.92,
+      "step": 2900,
+      "total_flos": 4.449947965854843e+17,
+      "train_loss": 1.5141178552035628,
+      "train_runtime": 27637.7455,
+      "train_samples_per_second": 6.74,
+      "train_steps_per_second": 0.105
+    }
+  ],
+  "logging_steps": 500,
+  "max_steps": 2900,
+  "num_train_epochs": 20,
+  "save_steps": 500,
+  "total_flos": 4.449947965854843e+17,
+  "trial_name": null,
+  "trial_params": null
+}