Add appy_chat_template process

spanish_medica_llm.py

@@ -403,6 +403,10 @@ def tokenize(element, tokenizer):
             input_batch.append(input_ids)
     return {"input_ids": input_batch}
 
+def apply_chat_template(example, tokenizer):
+    example['raw_text'] = tokenizer.apply_chat_template(example['raw_text'], tokenize=False)
+    return example
+
 def splitDatasetInTestValid(dataset):
     """
     """
@@ -705,8 +709,14 @@ def run_finnetuning_process():
     tokenizer = loadSpanishTokenizer()
     medicalSpanishDataset = applyChatInstructFormat( loadSpanishDatasetFinnetuning())
     print ( tokenizer.apply_chat_template(medicalSpanishDataset[5]['raw_text'], tokenize=False))
-    
+
     print('----------------------------------------------------------')
+    medicalSpanishDataset = medicalSpanishDataset.map(apply_chat_template,
+                                num_proc = os.cpu_count(),
+                                fn_kwargs = {'tokenizer':tokenizer},
+                                remove_columns = [col for col in medicalSpanishDataset.features if col not in ['raw_text']],
+                                desc = 'Applying chat template'
+                            )
     medicalSpanishDataset = tokenizer.apply_chat_template(medicalSpanishDataset, tokenize=False)
     medicalSpanishDataset = medicalSpanishDataset.train_test_split(0.2, seed=203984)
     train_dataset, eval_dataset, test_dataset = splitDatasetInTestValid(  medicalSpanishDataset )