Script for preparing training datasets

scripts.py

@@ -69,6 +69,39 @@ def train_base_model(task,filename):
     print(model.evaluate(test_data))
     model.save(filename)
     
+def prepare_training_datasets():
+    wikiqa = pandas.read_csv('corpora/WikiQA.csv')
+    avicenna = pandas.read_csv('corpora/Avicenna_Train.csv')
+    snli = pandas.read_csv('corpora/snli_1.0_train.csv')
+    question_answering = wikiqa.loc[wikiqa['Label']==1,
+                                    ['Cleaned_question',
+                                     'Resolved_answer']].rename(columns={'Cleaned_question':'question',
+                                                                         'Resolved_answer':'answer'})
+    reasoning = avicenna.loc[avicenna['Syllogistic relation']=='yes',
+                             ['Premise 1',
+                              'Premise 2',
+                              'Conclustion']].rename(columns={'Premise 1':'proposition0',
+                                                              'Premise 2':'proposition1',
+                                                              'Conclusion':'conclusion'})
+    consistency = snli[['sentence1',
+                        'sentence2']].rename(columns={'sentence1':'statement0',
+                                                      'sentence2':'statement1'})
+    mapping = {'entailment':1.0,
+               'neutral':0.0,
+               'contradiction':-1.0}
+    consistency['consistency'] = snli['gold_label'].apply(lambda x:mapping[x])
+    all_text = pandas.concatenate([wikiqa['Resolved_answer'],
+                                   avicenna['Premise 1'],
+                                   avicenna['Premise 1'],
+                                   reasoning['conclusion'],
+                                   snli['sentence1'],
+                                   snli['sentence2']]).to_frame(name='all_text')
+    all_text.to_csv('corpora/all_text.csv')
+    question_answering.to_csv('corpora/question_answering.csv')
+    reasoning.to_csv('corpora/reasoning_train.csv')
+    consistency.to_csv('corpora/consistency.csv')
+
+                                                              
     
     
     
@@ -84,4 +117,6 @@ if __name__ == '__main__':
         train_base_model(args.training_task,args.filename)
     elif args.task == 'prepare_wiki_qa':
         prepare_wiki_qa(args.filename,args.outputfile)
+    elif args.task == 'prepare_training_datasets':
+        prepare_training_datasets()