Spaces:
Sleeping
Sleeping
File size: 3,940 Bytes
c4e7d1d 16ae4af c4e7d1d fc903d0 c43fe93 c4e7d1d c43fe93 803836d 16ae4af b06fbc1 c43fe93 803836d f7da8c2 c43fe93 f7da8c2 803836d c43fe93 803836d c43fe93 803836d c43fe93 803836d c43fe93 803836d 0c119c4 803836d 3b31bf8 b14fd29 3b31bf8 b14fd29 c4e7d1d 16ae4af 803836d 0c119c4 803836d 0c119c4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 | import regex as re
try:
from config import config
module = config.language_identification.language_identification_library.lower()
except:
module = "langid"
langid_languages = ["af", "am", "an", "ar", "as", "az", "be", "bg", "bn", "br", "bs", "ca", "cs", "cy", "da", "de",
"dz", "el",
"en", "eo", "es", "et", "eu", "fa", "fi", "fo", "fr", "ga", "gl", "gu", "he", "hi", "hr", "ht",
"hu", "hy",
"id", "is", "it", "ja", "jv", "ka", "kk", "km", "kn", "ko", "ku", "ky", "la", "lb", "lo", "lt",
"lv", "mg",
"mk", "ml", "mn", "mr", "ms", "mt", "nb", "ne", "nl", "nn", "no", "oc", "or", "pa", "pl", "ps",
"pt", "qu",
"ro", "ru", "rw", "se", "si", "sk", "sl", "sq", "sr", "sv", "sw", "ta", "te", "th", "tl", "tr",
"ug", "uk",
"ur", "vi", "vo", "wa", "xh", "zh", "zu"]
classifier = None
def init_classifier():
global classifier
if module == "fastlid" or module == "fasttext":
from fastlid import fastlid
classifier = fastlid
elif module == "langid":
import langid
# init model
langid.langid.load_model()
classifier = langid.classify
init_classifier()
def set_languages(langs):
global classifier
if langs is None:
return
if module == "fastlid" or module == "fasttext":
from fastlid import fastlid, supported_langs
target_languages = [lang for lang in langs if lang in supported_langs]
fastlid.set_languages = target_languages
elif module == "langid":
import langid
target_languages = [lang for lang in langs if lang in langid_languages]
langid.set_languages(target_languages)
else:
raise ValueError(f"Wrong LANGUAGE_IDENTIFICATION_LIBRARY in config.yaml")
def classify_language(text: str, target_languages: list = None) -> str:
global classifier
if not target_languages:
target_languages = None
set_languages(target_languages)
lang = classifier(text)[0]
return lang
# def classify_zh_ja(text: str) -> str:
# for idx, char in enumerate(text):
# unicode_val = ord(char)
#
# # 检测日语字符
# if 0x3040 <= unicode_val <= 0x309F or 0x30A0 <= unicode_val <= 0x30FF:
# return "ja"
#
# # 检测汉字字符
# if 0x4E00 <= unicode_val <= 0x9FFF:
# # 检查周围的字符
# next_char = text[idx + 1] if idx + 1 < len(text) else None
#
# if next_char and (0x3040 <= ord(next_char) <= 0x309F or 0x30A0 <= ord(next_char) <= 0x30FF):
# return "ja"
#
# return "zh"
def split_alpha_nonalpha(text, mode=1):
"""
Splits the input text based on the specified mode.
Parameters:
- text (str): The input text to be split.
- mode (int): The mode for splitting (1 or 2).
- Mode 1: Splits based on the pattern - Chinese/Japanese followed by English or vice versa.
- Mode 2: Splits based on the pattern - Chinese/Japanese followed by English/digit or vice versa.
Returns:
- list: A list of substrings after the split.
"""
if mode == 1:
pattern = r'(?<=[\u4e00-\u9fff\u3040-\u30FF\d\s])(?=[\p{Latin}])|(?<=[\p{Latin}\s])(?=[\u4e00-\u9fff\u3040-\u30FF\d])'
elif mode == 2:
pattern = r'(?<=[\u4e00-\u9fff\u3040-\u30FF\s])(?=[\p{Latin}\d])|(?<=[\p{Latin}\d\s])(?=[\u4e00-\u9fff\u3040-\u30FF])'
else:
raise ValueError("Invalid mode. Supported modes are 1 and 2.")
return re.split(pattern, text)
if __name__ == "__main__":
text = "这是一个测试文本"
print(classify_language(text))
# print(classify_zh_ja(text)) # "zh"
text = "これはテストテキストです"
print(classify_language(text))
# print(classify_zh_ja(text)) # "ja"
|