Upload 7 files

korscideberta/normalize.py

@@ -0,0 +1,199 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import re
+import regex
+
+from itertools import chain
+
+
+class MosesPunctNormalizer:
+    """
+    This is a Python port of the Moses punctuation normalizer from
+    https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/normalize-punctuation.perl
+    """
+
+    EXTRA_WHITESPACE = [  # lines 21 - 30
+        (r"\r", r""),
+        (r"\(", r" ("),
+        (r"\)", r") "),
+        (r" +", r" "),
+        (r"\) ([.!:?;,])", r")\g<1>"),
+        (r"\( ", r"("),
+        (r" \)", r")"),
+        (r"(\d) %", r"\g<1>%"),
+        (r" :", r":"),
+        (r" ;", r";"),
+    ]
+
+    NORMALIZE_UNICODE_IF_NOT_PENN = [(r"`", r"'"), (r"''", r' " ')]  # lines 33 - 34
+
+    NORMALIZE_UNICODE = [  # lines 37 - 50
+        ("„", r'"'),
+        ("“", r'"'),
+        ("”", r'"'),
+        ("–", r"-"),
+        ("—", r" - "),
+        (r" +", r" "),
+        ("´", r"'"),
+        ("([a-zA-Z])‘([a-zA-Z])", r"\g<1>'\g<2>"),
+        ("([a-zA-Z])’([a-zA-Z])", r"\g<1>'\g<2>"),
+        ("‘", r"'"),
+        ("‚", r"'"),
+        ("’", r"'"),
+        (r"''", r'"'),
+        ("´´", r'"'),
+        ("…", r"..."),
+    ]
+
+    FRENCH_QUOTES = [  # lines 52 - 57
+        ("\u00A0«\u00A0", r'"'),
+        ("«\u00A0", r'"'),
+        ("«", r'"'),
+        ("\u00A0»\u00A0", r'"'),
+        ("\u00A0»", r'"'),
+        ("»", r'"'),
+    ]
+
+    HANDLE_PSEUDO_SPACES = [  # lines 59 - 67
+        ("\u00A0%", r"%"),
+        ("nº\u00A0", "nº "),
+        ("\u00A0:", r":"),
+        ("\u00A0ºC", " ºC"),
+        ("\u00A0cm", r" cm"),
+        ("\u00A0\\?", "?"),
+        ("\u00A0\\!", "!"),
+        ("\u00A0;", r";"),
+        (",\u00A0", r", "),
+        (r" +", r" "),
+    ]
+
+    EN_QUOTATION_FOLLOWED_BY_COMMA = [(r'"([,.]+)', r'\g<1>"')]
+
+    DE_ES_FR_QUOTATION_FOLLOWED_BY_COMMA = [
+        (r',"', r'",'),
+        (r'(\.+)"(\s*[^<])', r'"\g<1>\g<2>'),  # don't fix period at end of sentence
+    ]
+
+    DE_ES_CZ_CS_FR = [
+        ("(\\d)\u00A0(\\d)", r"\g<1>,\g<2>"),
+    ]
+
+    OTHER = [
+        ("(\\d)\u00A0(\\d)", r"\g<1>.\g<2>"),
+    ]
+
+    # Regex substitutions from replace-unicode-punctuation.perl
+    # https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/replace-unicode-punctuation.perl
+    REPLACE_UNICODE_PUNCTUATION = [
+        ("，", ","),
+        (r"。\s*", ". "),
+        ("、", ","),
+        ("”", '"'),
+        ("“", '"'),
+        ("∶", ":"),
+        ("：", ":"),
+        ("？", "?"),
+        ("《", '"'),
+        ("》", '"'),
+        ("）", ")"),
+        ("！", "!"),
+        ("（", "("),
+        ("；", ";"),
+        ("」", '"'),
+        ("「", '"'),
+        ("０", "0"),
+        ("１", "1"),
+        ("２", "2"),
+        ("３", "3"),
+        ("４", "4"),
+        ("５", "5"),
+        ("６", "6"),
+        ("７", "7"),
+        ("８", "8"),
+        ("９", "9"),
+        (r"．\s*", ". "),
+        ("～", "~"),
+        ("’", "'"),
+        ("…", "..."),
+        ("━", "-"),
+        ("〈", "<"),
+        ("〉", ">"),
+        ("【", "["),
+        ("】", "]"),
+        ("％", "%"),
+    ]
+
+    def __init__(
+        self,
+        lang="en",
+        penn=True,
+        norm_quote_commas=True,
+        norm_numbers=True,
+        pre_replace_unicode_punct=False,
+        post_remove_control_chars=False,
+    ):
+        """
+        :param language: The two-letter language code.
+        :type lang: str
+        :param penn: Normalize Penn Treebank style quotations.
+        :type penn: bool
+        :param norm_quote_commas: Normalize quotations and commas
+        :type norm_quote_commas: bool
+        :param norm_numbers: Normalize numbers
+        :type norm_numbers: bool
+        """
+        self.substitutions = [
+            self.EXTRA_WHITESPACE,
+            self.NORMALIZE_UNICODE,
+            self.FRENCH_QUOTES,
+            self.HANDLE_PSEUDO_SPACES,
+        ]
+
+        if penn:  # Adds the penn substitutions after extra_whitespace regexes.
+            self.substitutions.insert(1, self.NORMALIZE_UNICODE_IF_NOT_PENN)
+
+        if norm_quote_commas:
+            if lang == "en":
+                self.substitutions.append(self.EN_QUOTATION_FOLLOWED_BY_COMMA)
+            elif lang in ["de", "es", "fr"]:
+                self.substitutions.append(self.DE_ES_FR_QUOTATION_FOLLOWED_BY_COMMA)
+
+        if norm_numbers:
+            if lang in ["de", "es", "cz", "cs", "fr"]:
+                self.substitutions.append(self.DE_ES_CZ_CS_FR)
+            else:
+                self.substitutions.append(self.OTHER)
+
+        self.substitutions = list(chain(*self.substitutions))
+
+        self.pre_replace_unicode_punct = pre_replace_unicode_punct
+        self.post_remove_control_chars = post_remove_control_chars
+
+    def normalize(self, text):
+        """
+        Returns a string with normalized punctuation.
+        """
+        # Optionally, replace unicode puncts BEFORE normalization.
+        if self.pre_replace_unicode_punct:
+            text = self.replace_unicode_punct(text)
+
+        # Actual normalization.
+        for regexp, substitution in self.substitutions:
+            # print(regexp, substitution)
+            text = re.sub(regexp, substitution, str(text))
+            # print(text)
+
+        # Optionally, replace unicode puncts BEFORE normalization.
+        if self.post_remove_control_chars:
+            text = self.remove_control_chars(text)
+
+        return text.strip()
+
+    def replace_unicode_punct(self, text):
+        for regexp, substitution in self.REPLACE_UNICODE_PUNCTUATION:
+            text = re.sub(regexp, substitution, str(text))
+        return text
+
+    def remove_control_chars(self, text):
+        return regex.sub(r"\p{C}", "", text)

korscideberta/pyproject.toml

@@ -0,0 +1,20 @@
+# file: pyproject.toml
+
+[build-system]
+requires = [
+    "setuptools >= 65",
+    "wheel >= 0.38",
+]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "korscideberta"
+version = "0.1.0"
+readme = "README.md"
+requires-python = ">=3.8"
+dependencies = [
+    "sentencepiece",
+    "transformers",
+    "mecab",
+    "konlpy",
+]

korscideberta/tokenization_korscideberta.py

@@ -0,0 +1,357 @@
+# coding=utf-8
+# Copyright 2018 Google AI, Google Brain and Carnegie Mellon University Authors and the HuggingFace Inc. team and Jangwon Park
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" Tokenization classes for KoBERT model """
+
+
+import logging
+import os
+import re
+import unicodedata
+from shutil import copyfile
+
+from transformers import PreTrainedTokenizer
+
+#2023. 7. 28. 형태소 분리(Mecab), 유니코드 정규화 추가
+from konlpy.tag import Mecab
+from korscideberta.unicode import join_jamos
+from korscideberta.normalize import MosesPunctNormalizer
+nor = MosesPunctNormalizer()
+
+# 유니코드 한글 시작 : 44032, 끝 : 55199
+BASE_CODE, CHOSUNG, JUNGSUNG = 44032, 588, 28
+# 초성 리스트. 0 ~ 18
+CHOSUNG_LIST = ['ㄱ', 'ㄲ', 'ㄴ', 'ㄷ', 'ㄸ', 'ㄹ', 'ㅁ', 'ㅂ', 'ㅃ', 'ㅅ', 'ㅆ', 'ㅇ', 'ㅈ', 'ㅉ', 'ㅊ', 'ㅋ', 'ㅌ', 'ㅍ', 'ㅎ']
+# 중성 리스트. 0 ~ 20
+JUNGSUNG_LIST = ['ㅏ', 'ㅐ', 'ㅑ', 'ㅒ', 'ㅓ', 'ㅔ', 'ㅕ', 'ㅖ', 'ㅗ', 'ㅘ', 'ㅙ', 'ㅚ', 'ㅛ', 'ㅜ', 'ㅝ', 'ㅞ', 'ㅟ', 'ㅠ', 'ㅡ', 'ㅢ', 'ㅣ']
+# 종성 리스트. 0 ~ 27 + 1(1개 없음)
+JONGSUNG_LIST = [' ', 'ㄱ', 'ㄲ', 'ㄳ', 'ㄴ', 'ㄵ', 'ㄶ', 'ㄷ', 'ㄹ', 'ㄺ', 'ㄻ', 'ㄼ', 'ㄽ', 'ㄾ', 'ㄿ', 'ㅀ', 'ㅁ', 'ㅂ', 'ㅄ', 'ㅅ', 'ㅆ', 'ㅇ', 'ㅈ', 'ㅊ', 'ㅋ', 'ㅌ', 'ㅍ', 'ㅎ']
+def splitjamo(string):
+    sp_list = list(string)
+    result = []
+    for keyword in sp_list:
+        # 한글 여부 check 후 분리
+        if re.match('.*[ㄱ-ㅎㅏ-ㅣ가-힣]+.*', keyword) is not None:
+            # 초성
+            char_code = ord(keyword) - BASE_CODE
+            char1 = int(char_code / CHOSUNG)
+            try:
+                result.append(CHOSUNG_LIST[char1])
+            except:
+                return string
+                #print("Err: "+str(char1))
+            # 중성
+            char2 = int((char_code - (CHOSUNG * char1)) / JUNGSUNG)
+            result.append(JUNGSUNG_LIST[char2])
+            # 종성
+            char3 = int((char_code - (CHOSUNG * char1) - (JUNGSUNG * char2)))
+            result.append(JONGSUNG_LIST[char3])
+        else:
+            result.append(keyword)
+    return result
+def has_coda(word):
+    return (ord(word[-1]) -44032)%28==0
+def _replace_unicode(line):
+    if(line==None):
+        return ""
+    line = line.replace("—",'-').replace("―","-").replace("–","-").replace("＂",'"').replace("＇","'").replace("‹","<").replace("›",">").replace("‚","'").replace("‛","'").replace("„",'"').replace("‟",'"').replace("«",'<').replace("»",'>').replace("˝",'"').replace("（",'(').replace("）",')').replace("『",'"').replace("』",'"').replace("“",'"').replace("”",'"').replace("‘","'").replace("’","'").replace("《","<").replace("》",">").replace("〈","<").replace("〉",">").replace("「","'").replace("」","'").replace("【","[").replace("】","]").replace("〔","[").replace("〕","]").replace("［","[").replace("］","]").replace("｛","{").replace("｝","}")
+    line=nor.replace_unicode_punct(line)
+    return line
+def _mecab(line):
+    mecab = Mecab()
+    #참고: VV동사 VA형용사 VX보조 용언 VCP긍정 지정사 VCN부정 지정사 JKS주격 조사 JKC보격 조사, … XSN명사 파생 접미사 XSV동사 파생 접미사 XSA형용사 파생 접미사 EP선어말 어미 EF종결 어미 EC연결 어미 ETN명사형 전성 어미 ETM관형형 전성 어미
+    
+    pdoc = []
+    morphs = []
+    
+    poss = mecab.pos(line)
+    for pos in poss:
+        morphs.append(pos[0])
+    '''
+    pdoc.append(" ".join(morphs))
+    return pdoc
+    '''
+    return " ".join(morphs)
+
+logger = logging.getLogger(__name__)
+
+VOCAB_FILES_NAMES = {
+    "vocab_file": "spm.model",
+    "vocab_txt": "vocab.txt",
+}
+
+PRETRAINED_VOCAB_FILES_MAP = {
+    "vocab_file": {
+        "monologg/kobert": "https://s3.amazonaws.com/models.huggingface.co/bert/monologg/kobert/tokenizer_78b3253a26.model",
+        "monologg/kobert-lm": "https://s3.amazonaws.com/models.huggingface.co/bert/monologg/kobert-lm/tokenizer_78b3253a26.model",
+        "monologg/distilkobert": "https://s3.amazonaws.com/models.huggingface.co/bert/monologg/distilkobert/tokenizer_78b3253a26.model",
+    },
+    "vocab_txt": {
+        "monologg/kobert": "https://s3.amazonaws.com/models.huggingface.co/bert/monologg/kobert/vocab.txt",
+        "monologg/kobert-lm": "https://s3.amazonaws.com/models.huggingface.co/bert/monologg/kobert-lm/vocab.txt",
+        "monologg/distilkobert": "https://s3.amazonaws.com/models.huggingface.co/bert/monologg/distilkobert/vocab.txt",
+    },
+}
+
+PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
+    "monologg/kobert": 512,
+    "monologg/kobert-lm": 512,
+    "monologg/distilkobert": 512,
+}
+
+PRETRAINED_INIT_CONFIGURATION = {
+    "monologg/kobert": {"do_lower_case": False},
+    "monologg/kobert-lm": {"do_lower_case": False},
+    "monologg/distilkobert": {"do_lower_case": False},
+}
+
+SPIECE_UNDERLINE = "▁"
+
+
+class DebertaV2Tokenizer(PreTrainedTokenizer):
+    """
+    SentencePiece based tokenizer. Peculiarities:
+        - requires `SentencePiece <https://github.com/google/sentencepiece>`_
+    """
+
+    vocab_files_names = VOCAB_FILES_NAMES
+    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION
+    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
+
+    def __init__(
+        self,
+        vocab_file,
+        vocab_txt,
+        do_lower_case=False,
+        remove_space=True,
+        keep_accents=False,
+        unk_token="<unk>",
+        sep_token="<s>",
+        pad_token="<pad>",
+        cls_token="<cls>",
+        mask_token="<mask>",
+        **kwargs,
+    ):
+        super().__init__(
+            unk_token="<unk>",
+            sep_token=sep_token,
+            pad_token=pad_token,
+            cls_token=cls_token,
+            mask_token=mask_token,
+            **kwargs,
+        )
+
+        # Build vocab
+        self.token2idx = dict()
+        self.idx2token = []
+        with open(vocab_txt, "r", encoding="utf-8") as f:
+            for idx, token in enumerate(f):
+                token = token.strip()
+                self.token2idx[token] = idx
+                self.idx2token.append(token)
+
+        try:
+            import sentencepiece as spm
+        except ImportError:
+            logger.warning(
+                "You need to install SentencePiece to use KoBertTokenizer: https://github.com/google/sentencepiece"
+                "pip install sentencepiece"
+            )
+
+        self.do_lower_case = do_lower_case
+        self.remove_space = remove_space
+        self.keep_accents = keep_accents
+        self.vocab_file = vocab_file
+        self.vocab_txt = vocab_txt
+
+        self.sp_model = spm.SentencePieceProcessor()
+        self.sp_model.Load(vocab_file)
+
+    @property
+    def vocab_size(self):
+        return len(self.idx2token)
+
+    def get_vocab(self):
+        return dict(self.token2idx, **self.added_tokens_encoder)
+
+    def __getstate__(self):
+        state = self.__dict__.copy()
+        state["sp_model"] = None
+        return state
+
+    def __setstate__(self, d):
+        self.__dict__ = d
+        try:
+            import sentencepiece as spm
+        except ImportError:
+            logger.warning(
+                "You need to install SentencePiece to use KoBertTokenizer: https://github.com/google/sentencepiece"
+                "pip install sentencepiece"
+            )
+        self.sp_model = spm.SentencePieceProcessor()
+        self.sp_model.Load(self.vocab_file)
+
+    def preprocess_text(self, inputs):
+        if self.remove_space:
+            outputs = " ".join(inputs.strip().split())
+        else:
+            outputs = inputs
+        outputs = outputs.replace("``", '"').replace("''", '"')
+
+        if not self.keep_accents:
+            outputs = unicodedata.normalize("NFKD", outputs)
+            outputs = "".join([c for c in outputs if not unicodedata.combining(c)])
+        if self.do_lower_case:
+            outputs = outputs.lower()
+
+        return outputs
+
+    def _tokenize(self, text):
+        """Tokenize a string."""
+        text = self.preprocess_text(text)
+        #print('text: '+text)
+        #logger.info("text ({}) ".format(text))
+        text = _replace_unicode(text) #유니코드 정규화
+        text = _mecab(text) #형태소 분리
+        #print('text: '+str(text))
+        #logger.info("text ({}) ".format(text))
+        pieces = self.sp_model.encode(text, out_type=str)
+        new_pieces = []
+        for piece in pieces:
+            if len(piece) > 1 and piece[-1] == str(",") and piece[-2].isdigit():
+                cur_pieces = self.sp_model.EncodeAsPieces(piece[:-1].replace(SPIECE_UNDERLINE, ""))
+                if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
+                    if len(cur_pieces[0]) == 1:
+                        cur_pieces = cur_pieces[1:]
+                    else:
+                        cur_pieces[0] = cur_pieces[0][1:]
+                cur_pieces.append(piece[-1])
+                new_pieces.extend(cur_pieces)
+            else:
+                new_pieces.append(piece)
+        '''
+        return_pieces = []
+        for n in new_pieces:
+            if(isinstance(n,list)):
+                for nn in n:
+                    return_pieces.append(nn)
+            else:
+                return_pieces.append(n)
+        return return_pieces
+        '''
+        return new_pieces
+        
+
+    def _convert_token_to_id(self, token):
+        """ Converts a token (str/unicode) in an id using the vocab. """
+        return self.token2idx.get(token, self.token2idx[self.unk_token])
+
+    def _convert_id_to_token(self, index):
+        """Converts an index (integer) in a token (string/unicode) using the vocab."""
+        return self.idx2token[index]
+
+    def convert_tokens_to_string(self, tokens):
+        """Converts a sequence of tokens (strings for sub-words) in a single string."""
+        out_string = "".join(tokens).replace(SPIECE_UNDERLINE, " ").strip()
+        return out_string
+
+    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
+        """
+        Build model inputs from a sequence or a pair of sequence for sequence classification tasks
+        by concatenating and adding special tokens.
+        A KoBERT sequence has the following format:
+            single sequence: [CLS] X [SEP]
+            pair of sequences: [CLS] A [SEP] B [SEP]
+        """
+        if token_ids_1 is None:
+            return [self.cls_token_id] + token_ids_0 + [self.sep_token_id]
+        cls = [self.cls_token_id]
+        sep = [self.sep_token_id]
+        return cls + token_ids_0 + sep + token_ids_1 + sep
+
+    def get_special_tokens_mask(self, token_ids_0, token_ids_1=None, already_has_special_tokens=False):
+        """
+        Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding
+        special tokens using the tokenizer ``prepare_for_model`` or ``encode_plus`` methods.
+        Args:
+            token_ids_0: list of ids (must not contain special tokens)
+            token_ids_1: Optional list of ids (must not contain special tokens), necessary when fetching sequence ids
+                for sequence pairs
+            already_has_special_tokens: (default False) Set to True if the token list is already formated with
+                special tokens for the model
+        Returns:
+            A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token.
+        """
+
+        if already_has_special_tokens:
+            if token_ids_1 is not None:
+                raise ValueError(
+                    "You should not supply a second sequence if the provided sequence of "
+                    "ids is already formated with special tokens for the model."
+                )
+            return list(
+                map(
+                    lambda x: 1 if x in [self.sep_token_id, self.cls_token_id] else 0,
+                    token_ids_0,
+                )
+            )
+
+        if token_ids_1 is not None:
+            return [1] + ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1]
+        return [1] + ([0] * len(token_ids_0)) + [1]
+
+    def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None):
+        """
+        Creates a mask from the two sequences passed to be used in a sequence-pair classification task.
+        A KoBERT sequence pair mask has the following format:
+        0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1
+        | first sequence    | second sequence
+        if token_ids_1 is None, only returns the first portion of the mask (0's).
+        """
+        sep = [self.sep_token_id]
+        cls = [self.cls_token_id]
+        if token_ids_1 is None:
+            return len(cls + token_ids_0 + sep) * [0]
+        return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1]
+
+    def save_vocabulary(self, save_directory):
+        """Save the sentencepiece vocabulary (copy original file) and special tokens file
+        to a directory.
+        """
+        if not os.path.isdir(save_directory):
+            logger.error("Vocabulary path ({}) should be a directory".format(save_directory))
+            return
+
+        # 1. Save sentencepiece model
+        out_vocab_model = os.path.join(save_directory, VOCAB_FILES_NAMES["vocab_file"])
+
+        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_model):
+            copyfile(self.vocab_file, out_vocab_model)
+
+        # 2. Save vocab.txt
+        index = 0
+        out_vocab_txt = os.path.join(save_directory, VOCAB_FILES_NAMES["vocab_txt"])
+        with open(out_vocab_txt, "w", encoding="utf-8") as writer:
+            for token, token_index in sorted(self.token2idx.items(), key=lambda kv: kv[1]):
+                if index != token_index:
+                    logger.warning(
+                        "Saving vocabulary to {}: vocabulary indices are not consecutive."
+                        " Please check that the vocabulary is not corrupted!".format(out_vocab_txt)
+                    )
+                    index = token_index
+                writer.write(token + "\n")
+                index += 1
+
+        return out_vocab_model, out_vocab_txt

korscideberta/tokenization_korscideberta_v2.py

@@ -0,0 +1,580 @@
+# coding=utf-8
+# Copyright 2020 Microsoft and the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" Tokenization class for model DeBERTa."""
+
+import os
+import unicodedata
+from typing import Any, Dict, List, Optional, Tuple
+
+import sentencepiece as sp
+
+from transformers import AddedToken, PreTrainedTokenizer
+from transformers import logging
+#2023. 7. 28. 형태소 분리(Mecab), 유니코드 정규화 추가
+from konlpy.tag import Mecab
+from korscideberta.unicode import join_jamos
+from korscideberta.normalize import MosesPunctNormalizer
+nor = MosesPunctNormalizer()
+
+def has_coda(word):
+    return (ord(word[-1]) -44032)%28==0
+def _replace_unicode(line):
+    if(line==None):
+        return ""
+    line = line.replace("—",'-').replace("―","-").replace("–","-").replace("＂",'"').replace("＇","'").replace("‹","<").replace("›",">").replace("‚","'").replace("‛","'").replace("„",'"').replace("‟",'"').replace("«",'<').replace("»",'>').replace("˝",'"').replace("（",'(').replace("）",')').replace("『",'"').replace("』",'"').replace("“",'"').replace("”",'"').replace("‘","'").replace("’","'").replace("《","<").replace("》",">").replace("〈","<").replace("〉",">").replace("「","'").replace("」","'").replace("【","[").replace("】","]").replace("〔","[").replace("〕","]").replace("［","[").replace("］","]").replace("｛","{").replace("｝","}")
+    line=nor.replace_unicode_punct(line)
+    return line
+def _mecab(line):
+    mecab = Mecab()
+    #참고: VV동사 VA형용사 VX보조 용언 VCP긍정 지정사 VCN부정 지정사 JKS주격 조사 JKC보격 조사, … XSN명사 파생 접미사 XSV동사 파생 접미사 XSA형용사 파생 접미사 EP선어말 어미 EF종결 어미 EC연결 어미 ETN명사형 전성 어미 ETM관형형 전성 어미
+    
+    pdoc = []
+    morphs = []
+    
+    poss = mecab.pos(line)
+    for pos in poss:
+        morphs.append(pos[0])
+    '''
+    pdoc.append(" ".join(morphs))
+    return pdoc
+    '''
+    return " ".join(morphs)
+
+logger = logging.get_logger(__name__)
+
+PRETRAINED_VOCAB_FILES_MAP = {
+    "vocab_file": {
+        "microsoft/deberta-v2-xlarge": "https://huggingface.co/microsoft/deberta-v2-xlarge/resolve/main/spm.model",
+        "microsoft/deberta-v2-xxlarge": "https://huggingface.co/microsoft/deberta-v2-xxlarge/resolve/main/spm.model",
+        "microsoft/deberta-v2-xlarge-mnli": (
+            "https://huggingface.co/microsoft/deberta-v2-xlarge-mnli/resolve/main/spm.model"
+        ),
+        "microsoft/deberta-v2-xxlarge-mnli": (
+            "https://huggingface.co/microsoft/deberta-v2-xxlarge-mnli/resolve/main/spm.model"
+        ),
+    }
+}
+
+PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
+    "microsoft/deberta-v2-xlarge": 512,
+    "microsoft/deberta-v2-xxlarge": 512,
+    "microsoft/deberta-v2-xlarge-mnli": 512,
+    "microsoft/deberta-v2-xxlarge-mnli": 512,
+}
+
+PRETRAINED_INIT_CONFIGURATION = {
+    "microsoft/deberta-v2-xlarge": {"do_lower_case": False},
+    "microsoft/deberta-v2-xxlarge": {"do_lower_case": False},
+    "microsoft/deberta-v2-xlarge-mnli": {"do_lower_case": False},
+    "microsoft/deberta-v2-xxlarge-mnli": {"do_lower_case": False},
+}
+
+VOCAB_FILES_NAMES = {"vocab_file": "spm.model"}
+
+
+class DebertaV2Tokenizer(PreTrainedTokenizer):
+    r"""
+    Constructs a DeBERTa-v2 tokenizer. Based on [SentencePiece](https://github.com/google/sentencepiece).
+
+    Args:
+        vocab_file (`str`):
+            [SentencePiece](https://github.com/google/sentencepiece) file (generally has a *.spm* extension) that
+            contains the vocabulary necessary to instantiate a tokenizer.
+        do_lower_case (`bool`, *optional*, defaults to `False`):
+            Whether or not to lowercase the input when tokenizing.
+        bos_token (`string`, *optional*, defaults to `"[CLS]"`):
+            The beginning of sequence token that was used during pre-training. Can be used a sequence classifier token.
+            When building a sequence using special tokens, this is not the token that is used for the beginning of
+            sequence. The token used is the `cls_token`.
+        eos_token (`string`, *optional*, defaults to `"[SEP]"`):
+            The end of sequence token. When building a sequence using special tokens, this is not the token that is
+            used for the end of sequence. The token used is the `sep_token`.
+        unk_token (`str`, *optional*, defaults to `"[UNK]"`):
+            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
+            token instead.
+        sep_token (`str`, *optional*, defaults to `"[SEP]"`):
+            The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for
+            sequence classification or for a text and a question for question answering. It is also used as the last
+            token of a sequence built with special tokens.
+        pad_token (`str`, *optional*, defaults to `"[PAD]"`):
+            The token used for padding, for example when batching sequences of different lengths.
+        cls_token (`str`, *optional*, defaults to `"[CLS]"`):
+            The classifier token which is used when doing sequence classification (classification of the whole sequence
+            instead of per-token classification). It is the first token of the sequence when built with special tokens.
+        mask_token (`str`, *optional*, defaults to `"[MASK]"`):
+            The token used for masking values. This is the token used when training this model with masked language
+            modeling. This is the token which the model will try to predict.
+        sp_model_kwargs (`dict`, *optional*):
+            Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for
+            SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,
+            to set:
+
+            - `enable_sampling`: Enable subword regularization.
+            - `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.
+
+              - `nbest_size = {0,1}`: No sampling is performed.
+              - `nbest_size > 1`: samples from the nbest_size results.
+              - `nbest_size < 0`: assuming that nbest_size is infinite and samples from the all hypothesis (lattice)
+                using forward-filtering-and-backward-sampling algorithm.
+
+            - `alpha`: Smoothing parameter for unigram sampling, and dropout probability of merge operations for
+              BPE-dropout.
+    """
+
+    vocab_files_names = VOCAB_FILES_NAMES
+    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION
+    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
+
+    def __init__(
+        self,
+        vocab_file,
+        do_lower_case=False,
+        split_by_punct=False,
+        bos_token="[CLS]",
+        eos_token="[SEP]",
+        unk_token="[UNK]",
+        sep_token="[SEP]",
+        pad_token="[PAD]",
+        cls_token="[CLS]",
+        mask_token="[MASK]",
+        sp_model_kwargs: Optional[Dict[str, Any]] = None,
+        **kwargs,
+    ) -> None:
+        self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
+
+        if not os.path.isfile(vocab_file):
+            raise ValueError(
+                f"Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google pretrained"
+                " model use `tokenizer = AutoTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"
+            )
+        self.do_lower_case = do_lower_case
+        self.split_by_punct = split_by_punct
+        self.vocab_file = vocab_file
+        self._tokenizer = SPMTokenizer(
+            vocab_file, None, split_by_punct=split_by_punct, sp_model_kwargs=self.sp_model_kwargs
+        )
+        unk_token = AddedToken(unk_token, normalized=True, special=True) if isinstance(unk_token, str) else unk_token
+        super().__init__(
+            do_lower_case=do_lower_case,
+            bos_token=bos_token,
+            eos_token=eos_token,
+            unk_token=unk_token,
+            sep_token=sep_token,
+            pad_token=pad_token,
+            cls_token=cls_token,
+            mask_token=mask_token,
+            split_by_punct=split_by_punct,
+            sp_model_kwargs=self.sp_model_kwargs,
+            **kwargs,
+        )
+        self._tokenizer.special_tokens = self.all_special_tokens
+
+    @property
+    def vocab_size(self):
+        return len(self.vocab)
+
+    @property
+    def vocab(self):
+        return self._tokenizer.vocab
+
+    def get_vocab(self):
+        vocab = self.vocab.copy()
+        vocab.update(self.get_added_vocab())
+        return vocab
+
+    def _tokenize(self, text: str) -> List[str]:
+        """Take as input a string and return a list of strings (tokens) for words/sub-words"""
+        if self.do_lower_case:
+            text = text.lower()
+        return self._tokenizer.tokenize(text)
+
+    def _convert_token_to_id(self, token):
+        """Converts a token (str) in an id using the vocab."""
+        return self._tokenizer.spm.PieceToId(token)
+
+    def _convert_id_to_token(self, index):
+        """Converts an index (integer) in a token (str) using the vocab."""
+        return self._tokenizer.spm.IdToPiece(index) if index < self.vocab_size else self.unk_token
+
+    def convert_tokens_to_string(self, tokens):
+        """Converts a sequence of tokens (string) in a single string."""
+        return self._tokenizer.decode(tokens)
+
+    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
+        """
+        Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
+        adding special tokens. A DeBERTa sequence has the following format:
+
+        - single sequence: [CLS] X [SEP]
+        - pair of sequences: [CLS] A [SEP] B [SEP]
+
+        Args:
+            token_ids_0 (`List[int]`):
+                List of IDs to which the special tokens will be added.
+            token_ids_1 (`List[int]`, *optional*):
+                Optional second list of IDs for sequence pairs.
+
+        Returns:
+            `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.
+        """
+
+        if token_ids_1 is None:
+            return [self.cls_token_id] + token_ids_0 + [self.sep_token_id]
+        cls = [self.cls_token_id]
+        sep = [self.sep_token_id]
+        return cls + token_ids_0 + sep + token_ids_1 + sep
+
+    def get_special_tokens_mask(self, token_ids_0, token_ids_1=None, already_has_special_tokens=False):
+        """
+        Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding
+        special tokens using the tokenizer `prepare_for_model` or `encode_plus` methods.
+
+        Args:
+            token_ids_0 (`List[int]`):
+                List of IDs.
+            token_ids_1 (`List[int]`, *optional*):
+                Optional second list of IDs for sequence pairs.
+            already_has_special_tokens (`bool`, *optional*, defaults to `False`):
+                Whether or not the token list is already formatted with special tokens for the model.
+
+        Returns:
+            `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
+        """
+
+        if already_has_special_tokens:
+            return super().get_special_tokens_mask(
+                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
+            )
+
+        if token_ids_1 is not None:
+            return [1] + ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1]
+        return [1] + ([0] * len(token_ids_0)) + [1]
+
+    def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None):
+        """
+        Create a mask from the two sequences passed to be used in a sequence-pair classification task. A DeBERTa
+        sequence pair mask has the following format:
+
+        ```
+        0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
+        | first sequence    | second sequence |
+        ```
+
+        If `token_ids_1` is `None`, this method only returns the first portion of the mask (0s).
+
+        Args:
+            token_ids_0 (`List[int]`):
+                List of IDs.
+            token_ids_1 (`List[int]`, *optional*):
+                Optional second list of IDs for sequence pairs.
+
+        Returns:
+            `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
+        """
+        sep = [self.sep_token_id]
+        cls = [self.cls_token_id]
+        if token_ids_1 is None:
+            return len(cls + token_ids_0 + sep) * [0]
+        return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1]
+
+    def prepare_for_tokenization(self, text, is_split_into_words=False, **kwargs):
+        add_prefix_space = kwargs.pop("add_prefix_space", False)
+        if is_split_into_words or add_prefix_space:
+            text = " " + text
+        return (text, kwargs)
+
+    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
+        return self._tokenizer.save_pretrained(save_directory, filename_prefix=filename_prefix)
+
+
+class SPMTokenizer:
+    r"""
+    Constructs a tokenizer based on [SentencePiece](https://github.com/google/sentencepiece).
+
+    Args:
+        vocab_file (`str`):
+            [SentencePiece](https://github.com/google/sentencepiece) file (generally has a *.spm* extension) that
+            contains the vocabulary necessary to instantiate a tokenizer.
+        sp_model_kwargs (`dict`, *optional*):
+            Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for
+            SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,
+            to set:
+
+            - `enable_sampling`: Enable subword regularization.
+            - `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.
+
+              - `nbest_size = {0,1}`: No sampling is performed.
+              - `nbest_size > 1`: samples from the nbest_size results.
+              - `nbest_size < 0`: assuming that nbest_size is infinite and samples from the all hypothesis (lattice)
+                using forward-filtering-and-backward-sampling algorithm.
+
+            - `alpha`: Smoothing parameter for unigram sampling, and dropout probability of merge operations for
+              BPE-dropout.
+    """
+
+    def __init__(
+        self, vocab_file, special_tokens, split_by_punct=False, sp_model_kwargs: Optional[Dict[str, Any]] = None
+    ):
+        self.split_by_punct = split_by_punct
+        self.vocab_file = vocab_file
+        self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
+        spm = sp.SentencePieceProcessor(**self.sp_model_kwargs)
+        if not os.path.exists(vocab_file):
+            raise FileNotFoundError(f"{vocab_file} does not exist!")
+        spm.load(vocab_file)
+        bpe_vocab_size = spm.GetPieceSize()
+        # Token map
+        # <unk> 0+1
+        # <s> 1+1
+        # </s> 2+1
+        self.vocab = {spm.IdToPiece(i): i for i in range(bpe_vocab_size)}
+        self.ids_to_tokens = [spm.IdToPiece(i) for i in range(bpe_vocab_size)]
+        # self.vocab['[PAD]'] = 0
+        # self.vocab['[CLS]'] = 1
+        # self.vocab['[SEP]'] = 2
+        # self.vocab['[UNK]'] = 3
+
+        self.spm = spm
+        self.special_tokens = special_tokens
+
+    def __getstate__(self):
+        state = self.__dict__.copy()
+        state["spm"] = None
+        return state
+
+    def __setstate__(self, d):
+        self.__dict__ = d
+
+        # for backward compatibility
+        if not hasattr(self, "sp_model_kwargs"):
+            self.sp_model_kwargs = {}
+
+        self.spm = sp.SentencePieceProcessor(**self.sp_model_kwargs)
+        self.spm.Load(self.vocab_file)
+
+    def tokenize(self, text):
+        text = _replace_unicode(text) #유니코드 정규화
+        text = _mecab(text) #형태소 분리
+        return self._encode_as_pieces(text)
+
+    def convert_ids_to_tokens(self, ids):
+        tokens = []
+        for i in ids:
+            tokens.append(self.ids_to_tokens[i])
+        return tokens
+
+    def decode(self, tokens, start=-1, end=-1, raw_text=None):
+        if raw_text is None:
+            current_sub_tokens = []
+            out_string = ""
+            prev_is_special = False
+            for token in tokens:
+                # make sure that special tokens are not decoded using sentencepiece model
+                if token in self.special_tokens:
+                    if not prev_is_special:
+                        out_string += " "
+                    out_string += self.spm.decode_pieces(current_sub_tokens) + token
+                    prev_is_special = True
+                    current_sub_tokens = []
+                else:
+                    current_sub_tokens.append(token)
+                    prev_is_special = False
+            out_string += self.spm.decode_pieces(current_sub_tokens)
+            return out_string.strip()
+        else:
+            words = self.split_to_words(raw_text)
+            word_tokens = [self.tokenize(w) for w in words]
+            token2words = [0] * len(tokens)
+            tid = 0
+            for i, w in enumerate(word_tokens):
+                for k, t in enumerate(w):
+                    token2words[tid] = i
+                    tid += 1
+            word_start = token2words[start]
+            word_end = token2words[end] if end < len(tokens) else len(words)
+            text = "".join(words[word_start:word_end])
+            return text
+
+    # TODO add a deprecation cycle as this can have different behaviour from our API
+    def add_special_token(self, token):
+        if token not in self.special_tokens:
+            self.special_tokens.append(token)
+            if token not in self.vocab:
+                self.vocab[token] = len(self.vocab) - 1
+                self.ids_to_tokens.append(token)
+        return self.id(token)
+
+    def part_of_whole_word(self, token, is_bos=False):
+        logger.warning_once(
+            "The `DebertaTokenizer.part_of_whole_word` method is deprecated and will be removed in `transformers==4.35`"
+        )
+        if is_bos:
+            return True
+        if (
+            len(token) == 1
+            and (_is_whitespace(list(token)[0]) or _is_control(list(token)[0]) or _is_punctuation(list(token)[0]))
+        ) or token in self.special_tokens:
+            return False
+
+        word_start = b"\xe2\x96\x81".decode("utf-8")
+        return not token.startswith(word_start)
+
+    def pad(self):
+        return "[PAD]"
+
+    def bos(self):
+        return "[CLS]"
+
+    def eos(self):
+        return "[SEP]"
+
+    def unk(self):
+        return "[UNK]"
+
+    def mask(self):
+        return "[MASK]"
+
+    def sym(self, id):
+        return self.ids_to_tokens[id]
+
+    def id(self, sym):
+        logger.warning_once(
+            "The `DebertaTokenizer.id` method is deprecated and will be removed in `transformers==4.35`"
+        )
+        return self.vocab[sym] if sym in self.vocab else 1
+
+    def _encode_as_pieces(self, text):
+        text = convert_to_unicode(text)
+        if self.split_by_punct:
+            words = self._run_split_on_punc(text)
+            pieces = [self.spm.encode(w, out_type=str) for w in words]
+            return [p for w in pieces for p in w]
+        else:
+            return self.spm.encode(text, out_type=str)
+
+    def split_to_words(self, text):
+        pieces = self._encode_as_pieces(text)
+        word_start = b"\xe2\x96\x81".decode("utf-8")
+        words = []
+        offset = 0
+        prev_end = 0
+        for i, p in enumerate(pieces):
+            if p.startswith(word_start):
+                if offset > prev_end:
+                    words.append(text[prev_end:offset])
+                prev_end = offset
+                w = p.replace(word_start, "")
+            else:
+                w = p
+            try:
+                s = text.index(w, offset)
+                pn = ""
+                k = i + 1
+                while k < len(pieces):
+                    pn = pieces[k].replace(word_start, "")
+                    if len(pn) > 0:
+                        break
+                    k += 1
+
+                if len(pn) > 0 and pn in text[offset:s]:
+                    offset = offset + 1
+                else:
+                    offset = s + len(w)
+            except Exception:
+                offset = offset + 1
+
+        if prev_end < offset:
+            words.append(text[prev_end:offset])
+
+        return words
+
+    def _run_split_on_punc(self, text):
+        """Splits punctuation on a piece of text."""
+        chars = list(text)
+        i = 0
+        start_new_word = True
+        output = []
+        while i < len(chars):
+            char = chars[i]
+            if _is_punctuation(char):
+                output.append([char])
+                start_new_word = True
+            else:
+                if start_new_word:
+                    output.append([])
+                start_new_word = False
+                output[-1].append(char)
+            i += 1
+
+        return ["".join(x) for x in output]
+
+    def save_pretrained(self, path: str, filename_prefix: str = None):
+        filename = VOCAB_FILES_NAMES[list(VOCAB_FILES_NAMES.keys())[0]]
+        if filename_prefix is not None:
+            filename = filename_prefix + "-" + filename
+        full_path = os.path.join(path, filename)
+        with open(full_path, "wb") as fs:
+            fs.write(self.spm.serialized_model_proto())
+        return (full_path,)
+
+
+def _is_whitespace(char):
+    """Checks whether `chars` is a whitespace character."""
+    # \t, \n, and \r are technically control characters but we treat them
+    # as whitespace since they are generally considered as such.
+    if char == " " or char == "\t" or char == "\n" or char == "\r":
+        return True
+    cat = unicodedata.category(char)
+    if cat == "Zs":
+        return True
+    return False
+
+
+def _is_control(char):
+    """Checks whether `chars` is a control character."""
+    # These are technically control characters but we count them as whitespace
+    # characters.
+    if char == "\t" or char == "\n" or char == "\r":
+        return False
+    cat = unicodedata.category(char)
+    if cat.startswith("C"):
+        return True
+    return False
+
+
+def _is_punctuation(char):
+    """Checks whether `chars` is a punctuation character."""
+    cp = ord(char)
+    # We treat all non-letter/number ASCII as punctuation.
+    # Characters such as "^", "$", and "`" are not in the Unicode
+    # Punctuation class but we treat them as punctuation anyways, for
+    # consistency.
+    if (cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126):
+        return True
+    cat = unicodedata.category(char)
+    if cat.startswith("P"):
+        return True
+    return False
+
+
+def convert_to_unicode(text):
+    """Converts `text` to Unicode (if it's not already), assuming utf-8 input."""
+    if isinstance(text, str):
+        return text
+    elif isinstance(text, bytes):
+        return text.decode("utf-8", "ignore")
+    else:
+        raise ValueError(f"Unsupported string type: {type(text)}")

korscideberta/unicode.py

@@ -0,0 +1,279 @@
+__all__ = ["split_syllable_char", "split_syllables",
+           "join_jamos", "join_jamos_char",
+           "CHAR_INITIALS", "CHAR_MEDIALS", "CHAR_FINALS"]
+
+import itertools
+
+INITIAL = 0x001
+MEDIAL = 0x010
+FINAL = 0x100
+CHAR_LISTS = {
+    INITIAL: list(map(chr, [
+        0x3131, 0x3132, 0x3134, 0x3137, 0x3138, 0x3139,
+        0x3141, 0x3142, 0x3143, 0x3145, 0x3146, 0x3147,
+        0x3148, 0x3149, 0x314a, 0x314b, 0x314c, 0x314d,
+        0x314e
+    ])),
+    MEDIAL: list(map(chr, [
+        0x314f, 0x3150, 0x3151, 0x3152, 0x3153, 0x3154,
+        0x3155, 0x3156, 0x3157, 0x3158, 0x3159, 0x315a,
+        0x315b, 0x315c, 0x315d, 0x315e, 0x315f, 0x3160,
+        0x3161, 0x3162, 0x3163
+    ])),
+    FINAL: list(map(chr, [
+        0x3131, 0x3132, 0x3133, 0x3134, 0x3135, 0x3136,
+        0x3137, 0x3139, 0x313a, 0x313b, 0x313c, 0x313d,
+        0x313e, 0x313f, 0x3140, 0x3141, 0x3142, 0x3144,
+        0x3145, 0x3146, 0x3147, 0x3148, 0x314a, 0x314b,
+        0x314c, 0x314d, 0x314e
+    ]))
+}
+CHAR_INITIALS = CHAR_LISTS[INITIAL]
+CHAR_MEDIALS = CHAR_LISTS[MEDIAL]
+CHAR_FINALS = CHAR_LISTS[FINAL]
+CHAR_SETS = {k: set(v) for k, v in CHAR_LISTS.items()}
+CHARSET = set(itertools.chain(*CHAR_SETS.values()))
+CHAR_INDICES = {k: {c: i for i, c in enumerate(v)}
+                for k, v in CHAR_LISTS.items()}
+
+
+def is_hangul_syllable(c):
+    return 0xac00 <= ord(c) <= 0xd7a3  # Hangul Syllables
+
+
+def is_hangul_jamo(c):
+    return 0x1100 <= ord(c) <= 0x11ff  # Hangul Jamo
+
+
+def is_hangul_compat_jamo(c):
+    return 0x3130 <= ord(c) <= 0x318f  # Hangul Compatibility Jamo
+
+
+def is_hangul_jamo_exta(c):
+    return 0xa960 <= ord(c) <= 0xa97f  # Hangul Jamo Extended-A
+
+
+def is_hangul_jamo_extb(c):
+    return 0xd7b0 <= ord(c) <= 0xd7ff  # Hangul Jamo Extended-B
+
+
+def is_hangul(c):
+    return (is_hangul_syllable(c) or
+            is_hangul_jamo(c) or
+            is_hangul_compat_jamo(c) or
+            is_hangul_jamo_exta(c) or
+            is_hangul_jamo_extb(c))
+
+
+def is_supported_hangul(c):
+    return is_hangul_syllable(c) or is_hangul_compat_jamo(c)
+
+
+def check_hangul(c, jamo_only=False):
+    if not ((jamo_only or is_hangul_compat_jamo(c)) or is_supported_hangul(c)):
+        raise ValueError(f"'{c}' is not a supported hangul character. "
+                         f"'Hangul Syllables' (0xac00 ~ 0xd7a3) and "
+                         f"'Hangul Compatibility Jamos' (0x3130 ~ 0x318f) are "
+                         f"supported at the moment.")
+
+
+def get_jamo_type(c):
+    check_hangul(c)
+    assert is_hangul_compat_jamo(c), f"not a jamo: {ord(c):x}"
+    return sum(t for t, s in CHAR_SETS.items() if c in s)
+
+
+def split_syllable_char(c):
+    """
+    Splits a given korean syllable into its components. Each component is
+    represented by Unicode in 'Hangul Compatibility Jamo' range.
+
+    Arguments:
+        c: A Korean character.
+
+    Returns:
+        A triple (initial, medial, final) of Hangul Compatibility Jamos.
+        If no jamo corresponds to a position, `None` is returned there.
+
+    Example:
+        >>> split_syllable_char("안")
+        ("ㅇ", "ㅏ", "ㄴ")
+        >>> split_syllable_char("고")
+        ("ㄱ", "ㅗ", None)
+        >>> split_syllable_char("ㅗ")
+        (None, "ㅗ", None)
+        >>> split_syllable_char("ㅇ")
+        ("ㅇ", None, None)
+    """
+    check_hangul(c)
+    if len(c) != 1:
+        raise ValueError("Input string must have exactly one character.")
+
+    init, med, final = None, None, None
+    if is_hangul_syllable(c):
+        offset = ord(c) - 0xac00
+        x = (offset - offset % 28) // 28
+        init, med, final = x // 21, x % 21, offset % 28
+        if not final:
+            final = None
+        else:
+            final -= 1
+    else:
+        pos = get_jamo_type(c)
+        if pos & INITIAL == INITIAL:
+            pos = INITIAL
+        elif pos & MEDIAL == MEDIAL:
+            pos = MEDIAL
+        elif pos & FINAL == FINAL:
+            pos = FINAL
+        idx = CHAR_INDICES[pos][c]
+        if pos == INITIAL:
+            init = idx
+        elif pos == MEDIAL:
+            med = idx
+        else:
+            final = idx
+    return tuple(CHAR_LISTS[pos][idx] if idx is not None else None
+                 for pos, idx in
+                 zip([INITIAL, MEDIAL, FINAL], [init, med, final]))
+
+
+def split_syllables(s, ignore_err=True, pad=None):
+    """
+    Performs syllable-split on a string.
+
+    Arguments:
+        s (str): A string (possibly mixed with non-Hangul characters).
+        ignore_err (bool): If set False, it ensures that all characters in
+            the string are Hangul-splittable and throws a ValueError otherwise.
+            (default: True)
+        pad (str): Pad empty jamo positions (initial, medial, or final) with
+            `pad` character. This is useful for cases where fixed-length
+            strings are needed. (default: None)
+
+    Returns:
+        Hangul-split string
+
+    Example:
+        >>> split_syllables("안녕하세요")
+        "ㅇㅏㄴㄴㅕㅇ��ㅏㅅㅔㅇㅛ"
+        >>> split_syllables("안녕하세요~~", ignore_err=False)
+        ValueError: encountered an unsupported character: ~ (0x7e)
+        >>> split_syllables("안녕하세요ㅛ", pad="x")
+        'ㅇㅏㄴㄴㅕㅇㅎㅏxㅅㅔxㅇㅛxxㅛx'
+    """
+
+    def try_split(c):
+        try:
+            return split_syllable_char(c)
+        except ValueError:
+            if ignore_err:
+                return (c,)
+            raise ValueError(f"encountered an unsupported character: "
+                             f"{c} (0x{ord(c):x})")
+
+    s = map(try_split, s)
+    if pad is not None:
+        tuples = map(lambda x: tuple(pad if y is None else y for y in x), s)
+    else:
+        tuples = map(lambda x: filter(None, x), s)
+    return "".join(itertools.chain(*tuples))
+
+
+def join_jamos_char(init, med, final=None):
+    """
+    Combines jamos into a single syllable.
+
+    Arguments:
+        init (str): Initial jao.
+        med (str): Medial jamo.
+        final (str): Final jamo. If not supplied, the final syllable is made
+            without the final. (default: None)
+
+    Returns:
+        A Korean syllable.
+    """
+    chars = (init, med, final)
+    for c in filter(None, chars):
+        check_hangul(c, jamo_only=True)
+
+    idx = tuple(CHAR_INDICES[pos][c] if c is not None else c
+                for pos, c in zip((INITIAL, MEDIAL, FINAL), chars))
+    init_idx, med_idx, final_idx = idx
+    # final index must be shifted once as
+    # final index with 0 points to syllables without final
+    final_idx = 0 if final_idx is None else final_idx + 1
+    return chr(0xac00 + 28 * 21 * init_idx + 28 * med_idx + final_idx)
+
+
+def join_jamos(s, ignore_err=True):
+    """
+    Combines a sequence of jamos to produce a sequence of syllables.
+
+    Arguments:
+        s (str): A string (possible mixed with non-jamo characters).
+        ignore_err (bool): If set False, it will ensure that all characters
+            will be consumed for the making of syllables. It will throw a
+            ValueError when it fails to do so. (default: True)
+
+    Returns:
+        A string
+
+    Example:
+        >>> join_jamos("ㅇㅏㄴㄴㅕㅇㅎㅏㅅㅔㅇㅛ")
+        "안녕하세요"
+        >>> join_jamos("ㅇㅏㄴㄴㄴㅕㅇㅎㅏㅅㅔㅇㅛ")
+        "안ㄴ녕하세요"
+        >>> join_jamos()
+    """
+    last_t = 0
+    queue = []
+    new_string = ""
+
+    def flush(n=0):
+        new_queue = []
+        while len(queue) > n:
+            new_queue.append(queue.pop())
+        if len(new_queue) == 1:
+            if not ignore_err:
+                raise ValueError(f"invalid jamo character: {new_queue[0]}")
+            result = new_queue[0]
+        elif len(new_queue) >= 2:
+            try:
+                result = join_jamos_char(*new_queue)
+            except (ValueError, KeyError):
+                # Invalid jamo combination
+                if not ignore_err:
+                    raise ValueError(f"invalid jamo characters: {new_queue}")
+                result = "".join(new_queue)
+        else:
+            result = None
+        return result
+
+    for c in s:
+        if c not in CHARSET:
+            if queue:
+                new_c = flush() + c
+            else:
+                new_c = c
+            last_t = 0
+        else:
+            t = get_jamo_type(c)
+            new_c = None
+            if t & FINAL == FINAL:
+                if not (last_t == MEDIAL):
+                    new_c = flush()
+            elif t == INITIAL:
+                new_c = flush()
+            elif t == MEDIAL:
+                if last_t & INITIAL == INITIAL:
+                    new_c = flush(1)
+                else:
+                    new_c = flush()
+            last_t = t
+            queue.insert(0, c)
+        if new_c:
+            new_string += new_c
+    if queue:
+        new_string += flush()
+    return new_string

pyproject.toml

@@ -0,0 +1,20 @@
+# file: pyproject.toml
+
+[build-system]
+requires = [
+    "setuptools >= 65",
+    "wheel >= 0.38",
+]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "korscideberta"
+version = "0.1.0"
+readme = "README.md"
+requires-python = ">=3.8"
+dependencies = [
+    "sentencepiece",
+    "transformers",
+    "mecab",
+    "konlpy",
+]