tokenization_rnaernie.py

import os
from typing import Dict, List, Optional, Tuple

from transformers import PreTrainedTokenizer

_VOCAB = {
    "[PAD]": 0,
    "[UNK]": 1,
    "[CLS]": 2,
    "[SEP]": 3,
    "[MASK]": 4,
    "[DEL]": 5,
    "[IND]": 6,
    "RNaseMRPRNA": 7,
    "RNasePRNA": 8,
    "SRPRNA": 9,
    "YRNA": 10,
    "antisenseRNA": 11,
    "autocatalyticallysplicedintron": 12,
    "guideRNA": 13,
    "hammerheadribozyme": 14,
    "lncRNA": 15,
    "miRNA": 16,
    "miscRNA": 17,
    "ncRNA": 18,
    "other": 19,
    "piRNA": 20,
    "premiRNA": 21,
    "precursorRNA": 22,
    "rRNA": 23,
    "ribozyme": 24,
    "sRNA": 25,
    "scRNA": 26,
    "scaRNA": 27,
    "siRNA": 28,
    "snRNA": 29,
    "snoRNA": 30,
    "tRNA": 31,
    "telomeraseRNA": 32,
    "tmRNA": 33,
    "vaultRNA": 34,
    "A": 35,
    "T": 36,
    "C": 37,
    "G": 38,
}


class RNAErnieTokenizer(PreTrainedTokenizer):
    """Character-level RNA tokenizer for RNAErnie (original ERNIE/PaddlePaddle version).

    Converts U to T before tokenisation (model was pretrained with DNA-style T).
    Input sequences are uppercased and U->T substituted automatically.

    Vocabulary (39 tokens):
    - Special: [PAD]=0, [UNK]=1, [CLS]=2, [SEP]=3, [MASK]=4, [DEL]=5, [IND]=6
    - ncRNA type labels: indices 7-34 (28 labels)
    - Nucleotides: A=35, T=36, C=37, G=38
    """

    vocab_files_names = {"vocab_file": "vocab.txt"}
    model_input_names = ["input_ids", "attention_mask"]

    def __init__(
        self,
        vocab_file=None,
        pad_token="[PAD]",
        unk_token="[UNK]",
        cls_token="[CLS]",
        sep_token="[SEP]",
        mask_token="[MASK]",
        **kwargs,
    ):
        if vocab_file and os.path.isfile(vocab_file):
            self._vocab = {}
            with open(vocab_file, encoding="utf-8") as f:
                for idx, line in enumerate(f):
                    token = line.rstrip("\n")
                    self._vocab[token] = idx
        else:
            self._vocab = dict(_VOCAB)
        self._ids_to_tokens = {v: k for k, v in self._vocab.items()}
        super().__init__(
            pad_token=pad_token,
            unk_token=unk_token,
            cls_token=cls_token,
            sep_token=sep_token,
            mask_token=mask_token,
            **kwargs,
        )

    @property
    def vocab_size(self) -> int:
        return len(self._vocab)

    def get_vocab(self) -> Dict[str, int]:
        return dict(self._vocab)

    def _tokenize(self, text: str) -> List[str]:
        return list(text.upper().replace("U", "T"))

    def _convert_token_to_id(self, token: str) -> int:
        return self._vocab.get(token, self._vocab["[UNK]"])

    def _convert_id_to_token(self, index: int) -> str:
        return self._ids_to_tokens.get(index, "[UNK]")

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        os.makedirs(save_directory, exist_ok=True)
        fname = (filename_prefix + "-" if filename_prefix else "") + "vocab.txt"
        path = os.path.join(save_directory, fname)
        with open(path, "w", encoding="utf-8") as f:
            for token, _ in sorted(self._vocab.items(), key=lambda x: x[1]):
                f.write(token + "\n")
        return (path,)

    def build_inputs_with_special_tokens(self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None) -> List[int]:
        cls = [self.cls_token_id]
        sep = [self.sep_token_id]
        if token_ids_1 is None:
            return cls + token_ids_0 + sep
        return cls + token_ids_0 + sep + token_ids_1 + sep

    def get_special_tokens_mask(self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False) -> List[int]:
        if already_has_special_tokens:
            return super().get_special_tokens_mask(token_ids_0, token_ids_1, already_has_special_tokens=True)
        mask = [1] + [0] * len(token_ids_0) + [1]
        if token_ids_1 is not None:
            mask += [1] + [0] * len(token_ids_1) + [1]
        return mask

    def create_token_type_ids_from_sequences(self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None) -> List[int]:
        sep = [self.sep_token_id]
        cls = [self.cls_token_id]
        if token_ids_1 is None:
            return [0] * len(cls + token_ids_0 + sep)
        return [0] * len(cls + token_ids_0 + sep) + [1] * len(token_ids_1 + sep)