initial commit

README.md

@@ -0,0 +1,91 @@
+---
+language: tr
+datasets:
+- tquad1
+- tquad2
+- xquad 
+tags: 
+- text2text-generation
+- question-generation
+- answer-extraction
+- question-answering
+- text-generation
+pipeline_tag: text2text-generation
+widget:
+- text: "generate question: Legendary Entertainment, 2016 yılında bilimkurgu romanı Dune'un <hl> film ve TV haklarını <hl> satın aldı. Geliştirme kısa bir süre sonra başladı. Villeneuve projeye olan ilgisini dile getirdi ve resmi olarak yönetmen olarak imza attı. Roth ve Spaihts ile birlikte çalışarak senaryoyu iki bölüme ayırdı ve 1965 romanının 21. yüzyıla güncellenmiş bir uyarlamasını ekledi."
+  example_title: "Question Generation (Movie)"
+- text: "generate question: Fatih Sultan Mehmet, Cenevizlilerin önemli üslerinden Amasra’yı aldı. 1479’da <hl> bir antlaşma yaparak <hl> Venedik'le  16 yıllık savaşa son verdi."
+  example_title: "Question Generation (History)"
+- text: "generate question: Cenevizlilerin önemli üslerinden Amasra’yı aldı. 1479’da bir antlaşma yaparak <hl> Venedik'le <hl> 16 yıllık savaşa sona verdi."
+  example_title: "Question Generation (History 2)"
+- text: "extract answers: Cenevizlilerin önemli üslerinden Amasra’yı aldı. <hl> 1479’da bir antlaşma yaparak Venedik'le 16 yıllık savaşa sona verdi. <hl>"
+  example_title: "Answer Extraction (History)"
+- text: "question: Bu model ne ise yarar? context: Çalışmada sunulan yöntemle, Türkçe metinlerden otomatik olarak soru ve cevap üretilebilir. Bu proje ile paylaşılan kaynak kodu ile Türkçe Soru Üretme / Soru Cevaplama konularında yeni akademik çalışmalar yapılabilir. Projenin detaylarına paylaşılan Github ve Arxiv linklerinden ulaşılabilir."
+  example_title: "Answer Extraction (Open Domain)"
+license: cc-by-4.0
+---
+
+# mt5-small for Turkish Question Generation
+Automated question generation and question answering using text-to-text transformers by OBSS AI.
+```python
+from core.api import GenerationAPI
+generation_api = GenerationAPI('mt5-small-3task-highlight-tquad2')
+```
+
+##  Citation 📜
+```
+@article{akyon2021automated,
+  title={Automated question generation and question answering from Turkish texts using text-to-text transformers},
+  author={Akyon, Fatih Cagatay and Cavusoglu, Devrim and Cengiz, Cemil and Altinuc, Sinan Onur and Temizel, Alptekin},
+  journal={arXiv preprint arXiv:2111.06476},
+  year={2021}
+}
+```
+
+## Overview ✔️
+**Language model:** mt5-small  
+**Language:** Turkish  
+**Downstream-task:** Extractive QA/QG, Answer Extraction  
+**Training data:** TQuADv2-train
+**Code:**  https://github.com/obss/turkish-question-generation  
+**Paper:**  https://arxiv.org/abs/2111.06476  
+
+## Hyperparameters
+```
+batch_size = 256
+n_epochs = 15
+base_LM_model = "mt5-small"
+max_source_length = 512
+max_target_length = 64
+learning_rate = 1.0e-3
+task_lisst = ["qa", "qg", "ans_ext"]
+qg_format = "highlight"
+``` 
+
+## Performance
+Refer to [paper](https://arxiv.org/abs/2111.06476).
+
+## Usage 🔥
+```python
+from core.api import GenerationAPI
+
+generation_api = GenerationAPI('mt5-small-3task-highlight-tquad2')
+
+context = """
+Bu modelin eğitiminde, Türkçe soru cevap verileri kullanılmıştır.
+Çalışmada sunulan yöntemle, Türkçe metinlerden otomatik olarak soru ve cevap
+üretilebilir. Bu proje ile paylaşılan kaynak kodu ile Türkçe Soru Üretme
+/ Soru Cevaplama konularında yeni akademik çalışmalar yapılabilir.
+Projenin detaylarına paylaşılan Github ve Arxiv linklerinden ulaşılabilir.
+"""
+
+# a) Fully Automated Question Generation
+generation_api(task='question-generation', context=context)
+
+# b) Question Answering
+question = "Bu model ne işe yarar?"
+generation_api(task='question-answering', context=context, question=question)
+
+# b) Answer Extraction
+generation_api(task='answer-extraction', context=context)
+```

added_tokens.json

@@ -0,0 +1 @@
+{"<sep>": 250100, "<hl>": 250101}

config.json

@@ -0,0 +1,28 @@
+{
+  "_name_or_path": "google/mt5-small",
+  "architectures": [
+    "MT5ForConditionalGeneration"
+  ],
+  "d_ff": 1024,
+  "d_kv": 64,
+  "d_model": 512,
+  "decoder_start_token_id": 0,
+  "dropout_rate": 0.1,
+  "eos_token_id": 1,
+  "feed_forward_proj": "gated-gelu",
+  "initializer_factor": 1.0,
+  "is_encoder_decoder": true,
+  "layer_norm_epsilon": 1e-06,
+  "model_type": "mt5",
+  "num_decoder_layers": 8,
+  "num_heads": 6,
+  "num_layers": 8,
+  "pad_token_id": 0,
+  "relative_attention_num_buckets": 32,
+  "tie_word_embeddings": false,
+  "tokenizer_class": "T5Tokenizer",
+  "torch_dtype": "float32",
+  "transformers_version": "4.10.0.dev0",
+  "use_cache": true,
+  "vocab_size": 250102
+}

experiment_config.yaml

@@ -0,0 +1,106 @@
+_n_gpu: 1
+adafactor: false
+adam_beta1: 0.9
+adam_beta2: 0.999
+adam_epsilon: 1.0e-08
+cache_dir: null
+dataloader_drop_last: false
+dataloader_num_workers: 0
+dataloader_pin_memory: true
+ddp_find_unused_parameters: null
+debug: []
+deepspeed: null
+disable_tqdm: false
+do_eval: true
+do_predict: false
+do_train: true
+eval_accumulation_steps: 1
+eval_dataset_list:
+- tquad2-valid
+- xquad.tr
+eval_steps: 300
+evaluation_strategy: &id001 !!python/object/apply:transformers.trainer_utils.IntervalStrategy
+- steps
+fp16: false
+fp16_backend: auto
+fp16_full_eval: false
+fp16_opt_level: O1
+freeze_embeddings: false
+gradient_accumulation_steps: 4
+greater_is_better: null
+group_by_length: false
+ignore_data_skip: false
+label_names: null
+label_smoothing_factor: 0
+learning_rate: 0.001
+length_column_name: length
+load_best_model_at_end: false
+local_rank: -1
+log_level: -1
+log_level_replica: -1
+log_on_each_node: true
+logging_dir: runs/mt5-small/3task/adamw-1e3-15ep-highlight-tquad2train/runs/Sep06_02-59-11_palamut3.yonetim
+logging_first_step: false
+logging_steps: 500
+logging_strategy: *id001
+lr_scheduler_type: !!python/object/apply:transformers.trainer_utils.SchedulerType
+- linear
+max_grad_norm: 1.0
+max_source_length: 512
+max_steps: -1
+max_target_length: 64
+metric_for_best_model: null
+model_name_or_path: google/mt5-small
+model_type: mt5
+mp_parameters: ''
+mt5_qg_format: highlight
+mt5_task_list:
+- qa
+- qg
+- ans_ext
+neptune_api_token: eyJhcGlfYWRkcmVzcyI6Imh0dHBzOi8vYXBwLm5lcHR1bmUuYWkiLCJhcGlfdXJsIjoiaHR0cHM6Ly9hcHAubmVwdHVuZS5haSIsImFwaV9rZXkiOiJmMDMwZmQzYy01ZmQxLTQzZGMtYjYyOC02ZjUzMmMwYTkxZDgifQ==
+neptune_project: obss-ml/nqg-test
+neptune_run: null
+no_cuda: false
+num_train_epochs: 15
+output_dir: runs/mt5-small/3task/adamw-1e3-15ep-highlight-tquad2train
+overwrite_output_dir: false
+past_index: -1
+per_device_eval_batch_size: 64
+per_device_train_batch_size: 64
+per_gpu_eval_batch_size: null
+per_gpu_train_batch_size: null
+prediction_loss_only: false
+prepare_data: true
+push_to_hub: false
+push_to_hub_model_id: adamw-1e3-15ep-highlight-tquad2train
+push_to_hub_organization: null
+push_to_hub_token: null
+remove_unused_columns: false
+report_to:
+- wandb
+- neptune
+resume_from_checkpoint: null
+run_name: turque-mt5small-adamw-1e3-15ep-tquad2train
+save_on_each_node: false
+save_steps: 500
+save_strategy: *id001
+save_total_limit: 1
+seed: 42
+sharded_ddp: []
+skip_memory_metrics: true
+tokenizer_path: tokenizers/mt5-small
+tpu_metrics_debug: false
+tpu_num_cores: null
+train_dataset_list:
+- tquad2-train
+train_file_path: data/train_data.pt
+use_legacy_prediction_loop: false
+valid_dataset_list:
+- tquad2-valid
+valid_file_path: data/valid_data.pt
+wandb_id: null
+wandb_project: turkish-qa-qg
+warmup_ratio: 0.0
+warmup_steps: 0
+weight_decay: 0.0

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ccc5924aba16edb1effbe401ad0ad7a70883bbaa0e5935e885c9f323344e8dfd
+size 1200734941

special_tokens_map.json

@@ -0,0 +1 @@
+{"eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>"}

spiece.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef78f86560d809067d12bac6c09f19a462cb3af3f54d2b8acbba26e1433125d6
+size 4309802

tokenizer_config.json

@@ -0,0 +1 @@
+{"eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>", "extra_ids": 0, "additional_special_tokens": null, "special_tokens_map_file": "/truba/home/fakyon/.cache/huggingface/transformers/685ac0ca8568ec593a48b61b0a3c272beee9bc194a3c7241d15dcadb5f875e53.f76030f3ec1b96a8199b2593390c610e76ca8028ef3d24680000619ffb646276", "name_or_path": "tokenizers/mt5-small", "sp_model_kwargs": {}, "tokenizer_class": "T5Tokenizer"}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91ec8bb34db197dd105b1773420a72eb63f603c8da105943ab54385c7b177d27
+size 3119