Fix unsupported operand type(s) for |
Browse files
src/axolotl/prompt_tokenizers.py
CHANGED
|
@@ -4,7 +4,7 @@ import abc
|
|
| 4 |
import copy
|
| 5 |
import functools
|
| 6 |
import logging
|
| 7 |
-
from typing import Dict, List, Tuple
|
| 8 |
|
| 9 |
from transformers import PreTrainedTokenizer
|
| 10 |
|
|
@@ -433,7 +433,7 @@ def parse_tokenized_to_result(
|
|
| 433 |
current_len: int,
|
| 434 |
res: Dict[str, List[int]],
|
| 435 |
labels: list[int],
|
| 436 |
-
pad_token_id: int
|
| 437 |
) -> Tuple[Dict[str, List[int]], int]:
|
| 438 |
"""
|
| 439 |
Parses the tokenized prompt and append the tokenized input_ids, attention_mask and labels to the result
|
|
|
|
| 4 |
import copy
|
| 5 |
import functools
|
| 6 |
import logging
|
| 7 |
+
from typing import Dict, List, Tuple, Union
|
| 8 |
|
| 9 |
from transformers import PreTrainedTokenizer
|
| 10 |
|
|
|
|
| 433 |
current_len: int,
|
| 434 |
res: Dict[str, List[int]],
|
| 435 |
labels: list[int],
|
| 436 |
+
pad_token_id: Union[int, None] = None,
|
| 437 |
) -> Tuple[Dict[str, List[int]], int]:
|
| 438 |
"""
|
| 439 |
Parses the tokenized prompt and append the tokenized input_ids, attention_mask and labels to the result
|