|
|
| from typing import List, Optional
|
|
|
| from transformers import BertTokenizer
|
|
|
| from mmaction.registry import TOKENIZER
|
|
|
|
|
| class VindLUTokenizer(BertTokenizer):
|
| """VindLUTokenizer inherit BertTokenizer.
|
|
|
| The main difference from BertTokenizer is removing the last separate token
|
| for a single sequence.
|
| """
|
|
|
| def build_inputs_with_special_tokens(
|
| self,
|
| token_ids_0: List[int],
|
| token_ids_1: Optional[List[int]] = None) -> List[int]:
|
| """Build model inputs from a sequence or a pair of sequence for
|
| sequence classification tasks by concatenating and adding special
|
| tokens. A BERT sequence has the following format:
|
|
|
| - single sequence: `[CLS] X`
|
| - pair of sequences: `[CLS] A [SEP] B [SEP]`
|
|
|
| Args:
|
| token_ids_0 (`List[int]`):
|
| List of IDs to which the special tokens will be added.
|
| token_ids_1 (`List[int]`, *optional*):
|
| Optional second list of IDs for sequence pairs.
|
|
|
| Returns:
|
| `List[int]`: List of [input IDs](../glossary#input-ids) with
|
| the appropriate special tokens.
|
| """
|
| if token_ids_1 is None:
|
| return [self.cls_token_id] + token_ids_0
|
| cls = [self.cls_token_id]
|
| sep = [self.sep_token_id]
|
| return cls + token_ids_0 + sep + token_ids_1 + sep
|
|
|
|
|
| TOKENIZER.register_module(
|
| 'VindLUTokenizer', module=VindLUTokenizer.from_pretrained)
|
|
|
