Spaces:
Runtime error
Runtime error
Fix bug when call tokenize inner function
Browse files- spanish_medica_llm.py +1 -1
spanish_medica_llm.py
CHANGED
|
@@ -399,7 +399,7 @@ def getTokenizedDataset(dataset, tokenizer):
|
|
| 399 |
return dataset
|
| 400 |
|
| 401 |
return dataset.map(
|
| 402 |
-
tokenizer,
|
| 403 |
batched = True,
|
| 404 |
remove_columns = dataset["train"].column_names
|
| 405 |
)
|
|
|
|
| 399 |
return dataset
|
| 400 |
|
| 401 |
return dataset.map(
|
| 402 |
+
lambda element : tokenize(element, tokenizer) ,
|
| 403 |
batched = True,
|
| 404 |
remove_columns = dataset["train"].column_names
|
| 405 |
)
|