Removed train_base_models task. Not needed with RoBERTa models

scripts.py

@@ -6,7 +6,6 @@ import tokenizers
 import transformers
 import huggingface_hub
 import qarac.corpora.BNCorpus
-import qarac.corpora.Batcher
 import qarac.models.qarac_base_model
 import qarac.models.QaracTrainerModel
 import qarac.corpora.CombinedCorpus
@@ -72,30 +71,6 @@ def prepare_wiki_qa(filename,outfilename):
     data[['Cleaned_question','Resolved_answer','Label']].to_csv(outfilename)
 
         
-def train_base_model(task,filename):
-    tokenizer = tokenizers.Tokenizer.from_pretrained('xlm-roberta-base')
-    tokenizer.add_special_tokens(['<start>','<end>','<pad>'])
-    tokenizer.save('/'.join([os.environ['HOME'],
-                            'QARAC',
-                            'models',
-                            'tokenizer.json']))
-    bnc = qarac.corpora.BNCorpus.BNCorpus(tokenizer=tokenizer,
-                                          task=task)
-    (train,test)=bnc.split(0.01)
-    train_data=qarac.corpora.Batcher.Batcher(train)
-    model = qarac.models.qarac_base_model.qarac_base_model(tokenizer.get_vocab_size(), 
-                                                           768, 
-                                                           12,
-                                                           task=='decode')
-    #optimizer = keras.optimizers.Nadam(learning_rate=keras.optimizers.schedules.ExponentialDecay(1.0e-5, 100, 0.99))
-    #model.compile(optimizer=optimizer,loss='sparse_categorical_crossentropy',metrics='accuracy')
-    #model.fit(train_data,
-    #          epochs=100,
-    #          workers = 16,
-    #          use_multiprocessing=True)
-    test_data=qarac.corpora.Batcher.Batcher(test)
-    print(model.evaluate(test_data))
-    model.save(filename)
     
 def prepare_training_datasets():
     wikiqa = pandas.read_csv('corpora/WikiQA.csv')
@@ -478,9 +453,7 @@ if __name__ == '__main__':
     parser.add_argument('-t','--training-task')
     parser.add_argument('-o','--outputfile')
     args = parser.parse_args()
-    if args.task == 'train_base_model': 
-        train_base_model(args.training_task,args.filename)
-    elif args.task == 'prepare_wiki_qa':
+    if args.task == 'prepare_wiki_qa':
         prepare_wiki_qa(args.filename,args.outputfile)
     elif args.task == 'prepare_training_datasets':
         prepare_training_datasets()