model update

README.md

@@ -1,78 +1,404 @@
+
 ---
-language: 
-  - en
-tags:
-- question generation
-license: mit
-datasets:
-- asahi417/qg_squad
+license: cc-by-4.0
 metrics:
-- bleu
+- bleu4
 - meteor
-- rouge
+- rouge-l
 - bertscore
 - moverscore
+language: en
+datasets:
+- lmqg/qg_squad
+pipeline_tag: text2text-generation
+tags:
+- question generation
 widget:
-- text: "<hl> Beyonce <hl> further expanded her acting career, starring as blues singer Etta James in the 2008 musical biopic, Cadillac Records."
-  example_title: "Example 1"
-- text: "Beyonce further expanded her acting career, starring as blues singer <hl> Etta James <hl> in the 2008 musical biopic, Cadillac Records."
-  example_title: "Example 2"
-- text: "Beyonce further expanded her acting career, starring as blues singer Etta James in the 2008 musical biopic,  <hl> Cadillac Records  <hl> ."
-  example_title: "Example 3"
+- text: "generate question: <hl> Beyonce <hl> further expanded her acting career, starring as blues singer Etta James in the 2008 musical biopic, Cadillac Records."
+  example_title: "Question Generation Example 1" 
+- text: "generate question: Beyonce further expanded her acting career, starring as blues singer <hl> Etta James <hl> in the 2008 musical biopic, Cadillac Records."
+  example_title: "Question Generation Example 2" 
+- text: "generate question: Beyonce further expanded her acting career, starring as blues singer Etta James in the 2008 musical biopic,  <hl> Cadillac Records <hl> ."
+  example_title: "Question Generation Example 3" 
+model-index:
+- name: lmqg/bart-large-squad
+  results:
+  - task:
+      name: Text2text Generation
+      type: text2text-generation
+    dataset:
+      name: lmqg/qg_squad
+      type: default
+      args: default
+    metrics:
+    - name: BLEU4
+      type: bleu4
+      value: 0.26168385362299557
+    - name: ROUGE-L
+      type: rouge-l
+      value: 0.5384959163821219
+    - name: METEOR
+      type: meteor
+      value: 0.27073122286541956
+    - name: BERTScore
+      type: bertscore
+      value: 0.9100413219045603
+    - name: MoverScore
+      type: moverscore
+      value: 0.6499011626820898
+  - task:
+      name: Text2text Generation
+      type: text2text-generation
+    dataset:
+      name: lmqg/qg_squadshifts
+      type: reddit
+      args: reddit
+    metrics:
+    - name: BLEU4
+      type: bleu4
+      value: 0.059525104157825456
+    - name: ROUGE-L
+      type: rouge-l
+      value: 0.22365090580055863
+    - name: METEOR
+      type: meteor
+      value: 0.21499800504546457
+    - name: BERTScore
+      type: bertscore
+      value: 0.9095144685254328
+    - name: MoverScore
+      type: moverscore
+      value: 0.6059332247878408
+  - task:
+      name: Text2text Generation
+      type: text2text-generation
+    dataset:
+      name: lmqg/qg_squadshifts
+      type: new_wiki
+      args: new_wiki
+    metrics:
+    - name: BLEU4
+      type: bleu4
+      value: 0.11118273173452982
+    - name: ROUGE-L
+      type: rouge-l
+      value: 0.2967546690273089
+    - name: METEOR
+      type: meteor
+      value: 0.27315087810722966
+    - name: BERTScore
+      type: bertscore
+      value: 0.9322739617807421
+    - name: MoverScore
+      type: moverscore
+      value: 0.6623000084761579
+  - task:
+      name: Text2text Generation
+      type: text2text-generation
+    dataset:
+      name: lmqg/qg_subjqa
+      type: tripadvisor
+      args: tripadvisor
+    metrics:
+    - name: BLEU4
+      type: bleu4
+      value: 8.380171318718442e-07
+    - name: ROUGE-L
+      type: rouge-l
+      value: 0.1402922852924756
+    - name: METEOR
+      type: meteor
+      value: 0.1372146070365174
+    - name: BERTScore
+      type: bertscore
+      value: 0.8891002409937424
+    - name: MoverScore
+      type: moverscore
+      value: 0.5604572211470809
+  - task:
+      name: Text2text Generation
+      type: text2text-generation
+    dataset:
+      name: lmqg/qg_squadshifts
+      type: default
+      args: default
+    metrics:
+    - name: BLEU4
+      type: bleu4
+      value: 0.07839941048417529
+    - name: ROUGE-L
+      type: rouge-l
+      value: 0.25357667226247294
+    - name: METEOR
+      type: meteor
+      value: 0.24046838149047955
+    - name: BERTScore
+      type: bertscore
+      value: 0.9182198703598111
+    - name: MoverScore
+      type: moverscore
+      value: 0.6274693859765924
+  - task:
+      name: Text2text Generation
+      type: text2text-generation
+    dataset:
+      name: lmqg/qg_squadshifts
+      type: nyt
+      args: nyt
+    metrics:
+    - name: BLEU4
+      type: bleu4
+      value: 0.08117757543966063
+    - name: ROUGE-L
+      type: rouge-l
+      value: 0.25292097720734297
+    - name: METEOR
+      type: meteor
+      value: 0.25254205113198686
+    - name: BERTScore
+      type: bertscore
+      value: 0.9249009759439454
+    - name: MoverScore
+      type: moverscore
+      value: 0.6406329128556304
+  - task:
+      name: Text2text Generation
+      type: text2text-generation
+    dataset:
+      name: lmqg/qg_subjqa
+      type: restaurants
+      args: restaurants
+    metrics:
+    - name: BLEU4
+      type: bleu4
+      value: 1.1301750984972448e-06
+    - name: ROUGE-L
+      type: rouge-l
+      value: 0.13083168975354642
+    - name: METEOR
+      type: meteor
+      value: 0.12419733006916912
+    - name: BERTScore
+      type: bertscore
+      value: 0.8797711839570719
+    - name: MoverScore
+      type: moverscore
+      value: 0.5542757411268555
+  - task:
+      name: Text2text Generation
+      type: text2text-generation
+    dataset:
+      name: lmqg/qg_subjqa
+      type: electronics
+      args: electronics
+    metrics:
+    - name: BLEU4
+      type: bleu4
+      value: 0.00866799444965211
+    - name: ROUGE-L
+      type: rouge-l
+      value: 0.1601628874804186
+    - name: METEOR
+      type: meteor
+      value: 0.15348605312210778
+    - name: BERTScore
+      type: bertscore
+      value: 0.8783386920680519
+    - name: MoverScore
+      type: moverscore
+      value: 0.5634845371093992
+  - task:
+      name: Text2text Generation
+      type: text2text-generation
+    dataset:
+      name: lmqg/qg_subjqa
+      type: books
+      args: books
+    metrics:
+    - name: BLEU4
+      type: bleu4
+      value: 0.006278914808207679
+    - name: ROUGE-L
+      type: rouge-l
+      value: 0.12368226019088967
+    - name: METEOR
+      type: meteor
+      value: 0.11576293675813865
+    - name: BERTScore
+      type: bertscore
+      value: 0.8807110440044503
+    - name: MoverScore
+      type: moverscore
+      value: 0.5555905941686486
+  - task:
+      name: Text2text Generation
+      type: text2text-generation
+    dataset:
+      name: lmqg/qg_subjqa
+      type: movies
+      args: movies
+    metrics:
+    - name: BLEU4
+      type: bleu4
+      value: 1.0121579426501661e-06
+    - name: ROUGE-L
+      type: rouge-l
+      value: 0.12508697028506718
+    - name: METEOR
+      type: meteor
+      value: 0.11862284941640638
+    - name: BERTScore
+      type: bertscore
+      value: 0.8748829724726739
+    - name: MoverScore
+      type: moverscore
+      value: 0.5528899173535703
+  - task:
+      name: Text2text Generation
+      type: text2text-generation
+    dataset:
+      name: lmqg/qg_subjqa
+      type: grocery
+      args: grocery
+    metrics:
+    - name: BLEU4
+      type: bleu4
+      value: 0.00528043272450429
+    - name: ROUGE-L
+      type: rouge-l
+      value: 0.12343711316491492
+    - name: METEOR
+      type: meteor
+      value: 0.15133496445452477
+    - name: BERTScore
+      type: bertscore
+      value: 0.8778951253890991
+    - name: MoverScore
+      type: moverscore
+      value: 0.5701949938103265
+  - task:
+      name: Text2text Generation
+      type: text2text-generation
+    dataset:
+      name: lmqg/qg_squadshifts
+      type: amazon
+      args: amazon
+    metrics:
+    - name: BLEU4
+      type: bleu4
+      value: 0.06530369842068952
+    - name: ROUGE-L
+      type: rouge-l
+      value: 0.25030985091008146
+    - name: METEOR
+      type: meteor
+      value: 0.2229994442645732
+    - name: BERTScore
+      type: bertscore
+      value: 0.9092814804525936
+    - name: MoverScore
+      type: moverscore
+      value: 0.6086538514008419
+  - task:
+      name: Text2text Generation
+      type: text2text-generation
+    dataset:
+      name: lmqg/qg_subjqa
+      type: default
+      args: default
+    metrics:
+    - name: BLEU4
+      type: bleu4
+      value: 0.005121882223046874
+    - name: ROUGE-L
+      type: rouge-l
+      value: 0.1346485324169255
+    - name: METEOR
+      type: meteor
+      value: 0.13733272662214893
+    - name: BERTScore
+      type: bertscore
+      value: 0.8811488576438816
+    - name: MoverScore
+      type: moverscore
+      value: 0.5614233235005509
 ---
 
-# BART LARGE fine-tuned for English Question Generation
-BART LARGE Model fine-tuned on English question generation dataset (SQuAD) with an extensive hyper-parameter search.
-- [Online Demo](https://autoqg.net/)
-- [Project Repository](https://github.com/asahi417/lm-question-generation)
+# Language Models Fine-tuning on Question Generation: `lmqg/bart-large-squad`
+This model is fine-tuned version of [facebook/bart-large](https://huggingface.co/facebook/bart-large) for question generation task on the 
+[lmqg/qg_squad](https://huggingface.co/datasets/lmqg/qg_squad) (dataset_name: default).
 
-## Overview
 
-**Language model:** facebook/bart-large
-**Language:** English (en)    
-**Downstream-task:** Question Generation  
-**Training data:** SQuAD  
-**Eval data:** SQuAD   
-**Code:**  See [our repository](https://github.com/asahi417/lm-question-generation)
+### Overview
+- **Language model:** [facebook/bart-large](https://huggingface.co/facebook/bart-large)   
+- **Language:** en  
+- **Training data:** [lmqg/qg_squad](https://huggingface.co/datasets/lmqg/qg_squad) (default)
+- **Online Demo:** [https://autoqg.net/](https://autoqg.net/)
+- **Repository:** [https://github.com/asahi417/lm-question-generation](https://github.com/asahi417/lm-question-generation)
+- **Paper:** [TBA](TBA)
 
-## Usage
-### In Transformers
+### Usage
 ```python
+
 from transformers import pipeline
 
-model_path = 'asahi417/lmqg-bart-large-squad'
+model_path = 'lmqg/bart-large-squad'
 pipe = pipeline("text2text-generation", model_path)
 
-paragraph = 'Beyonce further expanded her acting career, starring as blues singer Etta James in the 2008 musical biopic, Cadillac Records.'
-# highlight an answer in the paragraph to generate question
-answer = 'Etta James'
-highlight_token = '<hl>'
-input_text = paragraph.replace(answer, '{0} {1} {0}'.format(highlight_token, answer))
-input_text = 'generate question: {}'.format(input_text)  # add task specific prefix
-generation = pipe(input_text)
-print(generation)
->>> [{'generated_text': 'What is the name of the biopic that Beyonce starred in?'}]
+# Question Generation
+input_text = 'generate question: <hl> Beyonce <hl> further expanded her acting career, starring as blues singer Etta James in the 2008 musical biopic, Cadillac Records.'
+question = pipe(input_text)
 ```
 
-## Evaluations
+## Evaluation Metrics
 
-Evaluation on the test set of [SQuAD QG dataset](https://huggingface.co/datasets/asahi417/qg_squad).
-The results are comparable with the [leaderboard](https://paperswithcode.com/sota/question-generation-on-squad11) and previous works.
-All evaluations were done using our [evaluation script](https://github.com/asahi417/lm-question-generation).
 
+### Metrics
 
-| BLEU 4 | ROUGE L  | METEOR | BERTScore | MoverScore |
-| ------ | -------- | ------ | --------- | ---------- |
-| 26.16  | 53.84    | 27.07  | 91.00     | 64.99      |
+| Dataset | Type | BLEU4 | ROUGE-L | METEOR | BERTScore | MoverScore | Link |
+|:--------|:-----|------:|--------:|-------:|----------:|-----------:|-----:|
+| [lmqg/qg_squad](https://huggingface.co/datasets/lmqg/qg_squad) | default | 0.26168385362299557 | 0.5384959163821219 | 0.27073122286541956 | 0.9100413219045603 | 0.6499011626820898 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval/metric.first.sentence.paragraph_answer.question.lmqg_qg_squad.default.json) | 
 
-- [metric file](https://huggingface.co/asahi417/lmqg-bart-large-squad/raw/main/eval/metric.first.sentence.paragraph_answer.question.asahi417_qg_squad.default.json)
 
 
-## Fine-tuning Parameters
-We ran grid search to find the best hyper-parameters and continued fine-tuning until the validation metric decrease.
-The best hyper-parameters can be found [here](https://huggingface.co/asahi417/lmqg-bart-large-squad/raw/main/trainer_config.json), and fine-tuning script is released in [our  repository](https://github.com/asahi417/lm-question-generation).
+### Out-of-domain Metrics
+        
+| Dataset | Type | BLEU4 | ROUGE-L | METEOR | BERTScore | MoverScore | Link |
+|:--------|:-----|------:|--------:|-------:|----------:|-----------:|-----:|
+| [lmqg/qg_squadshifts](https://huggingface.co/datasets/lmqg/qg_squadshifts) | reddit | 0.059525104157825456 | 0.22365090580055863 | 0.21499800504546457 | 0.9095144685254328 | 0.6059332247878408 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval_ood/metric.first.sentence.paragraph_answer.question.lmqg_qg_squadshifts.reddit.json) |
+| [lmqg/qg_squadshifts](https://huggingface.co/datasets/lmqg/qg_squadshifts) | new_wiki | 0.11118273173452982 | 0.2967546690273089 | 0.27315087810722966 | 0.9322739617807421 | 0.6623000084761579 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval_ood/metric.first.sentence.paragraph_answer.question.lmqg_qg_squadshifts.new_wiki.json) |
+| [lmqg/qg_subjqa](https://huggingface.co/datasets/lmqg/qg_subjqa) | tripadvisor | 8.380171318718442e-07 | 0.1402922852924756 | 0.1372146070365174 | 0.8891002409937424 | 0.5604572211470809 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval_ood/metric.first.sentence.paragraph_answer.question.lmqg_qg_subjqa.tripadvisor.json) |
+| [lmqg/qg_squadshifts](https://huggingface.co/datasets/lmqg/qg_squadshifts) | default | 0.07839941048417529 | 0.25357667226247294 | 0.24046838149047955 | 0.9182198703598111 | 0.6274693859765924 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval_ood/metric.first.sentence.paragraph_answer.question.lmqg_qg_squadshifts.default.json) |
+| [lmqg/qg_squadshifts](https://huggingface.co/datasets/lmqg/qg_squadshifts) | nyt | 0.08117757543966063 | 0.25292097720734297 | 0.25254205113198686 | 0.9249009759439454 | 0.6406329128556304 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval_ood/metric.first.sentence.paragraph_answer.question.lmqg_qg_squadshifts.nyt.json) |
+| [lmqg/qg_subjqa](https://huggingface.co/datasets/lmqg/qg_subjqa) | restaurants | 1.1301750984972448e-06 | 0.13083168975354642 | 0.12419733006916912 | 0.8797711839570719 | 0.5542757411268555 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval_ood/metric.first.sentence.paragraph_answer.question.lmqg_qg_subjqa.restaurants.json) |
+| [lmqg/qg_subjqa](https://huggingface.co/datasets/lmqg/qg_subjqa) | electronics | 0.00866799444965211 | 0.1601628874804186 | 0.15348605312210778 | 0.8783386920680519 | 0.5634845371093992 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval_ood/metric.first.sentence.paragraph_answer.question.lmqg_qg_subjqa.electronics.json) |
+| [lmqg/qg_subjqa](https://huggingface.co/datasets/lmqg/qg_subjqa) | books | 0.006278914808207679 | 0.12368226019088967 | 0.11576293675813865 | 0.8807110440044503 | 0.5555905941686486 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval_ood/metric.first.sentence.paragraph_answer.question.lmqg_qg_subjqa.books.json) |
+| [lmqg/qg_subjqa](https://huggingface.co/datasets/lmqg/qg_subjqa) | movies | 1.0121579426501661e-06 | 0.12508697028506718 | 0.11862284941640638 | 0.8748829724726739 | 0.5528899173535703 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval_ood/metric.first.sentence.paragraph_answer.question.lmqg_qg_subjqa.movies.json) |
+| [lmqg/qg_subjqa](https://huggingface.co/datasets/lmqg/qg_subjqa) | grocery | 0.00528043272450429 | 0.12343711316491492 | 0.15133496445452477 | 0.8778951253890991 | 0.5701949938103265 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval_ood/metric.first.sentence.paragraph_answer.question.lmqg_qg_subjqa.grocery.json) |
+| [lmqg/qg_squadshifts](https://huggingface.co/datasets/lmqg/qg_squadshifts) | amazon | 0.06530369842068952 | 0.25030985091008146 | 0.2229994442645732 | 0.9092814804525936 | 0.6086538514008419 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval_ood/metric.first.sentence.paragraph_answer.question.lmqg_qg_squadshifts.amazon.json) |
+| [lmqg/qg_subjqa](https://huggingface.co/datasets/lmqg/qg_subjqa) | default | 0.005121882223046874 | 0.1346485324169255 | 0.13733272662214893 | 0.8811488576438816 | 0.5614233235005509 | [link](https://huggingface.co/lmqg/bart-large-squad/raw/main/eval_ood/metric.first.sentence.paragraph_answer.question.lmqg_qg_subjqa.default.json) |
 
-## Citation
-TBA
 
+## Training hyperparameters
+
+The following hyperparameters were used during fine-tuning:
+ - dataset_path: lmqg/qg_squad
+ - dataset_name: default
+ - input_types: ['paragraph_answer']
+ - output_types: ['question']
+ - prefix_types: None
+ - model: facebook/bart-large
+ - max_length: 512
+ - max_length_output: 32
+ - epoch: 4
+ - batch: 32
+ - lr: 5e-05
+ - fp16: False
+ - random_seed: 1
+ - gradient_accumulation_steps: 4
+ - label_smoothing: 0.15
 
+The full configuration can be found at [fine-tuning config file](https://huggingface.co/lmqg/bart-large-squad/raw/main/trainer_config.json).
+
+## Citation
+TBA