Update tools/sentence.py

tools/sentence.py

@@ -107,6 +107,8 @@ def remove_numeric_annotations(text):
     pattern = r'“\d+”|【\d+】|〔\d+〕'
     # 使用正则表达式替换掉这些注释
     cleaned_text = re.sub(pattern, '', text)
+    cleaned_text = re.sub('「', '', cleaned_text)
+    cleaned_text = re.sub('」', '', cleaned_text)
     return cleaned_text
 
 def merge_adjacent_japanese(sentences):
@@ -128,11 +130,14 @@ def extrac(text):
     text = replace_quotes(remove_numeric_annotations(text))  # 替换引号
     text = re.sub("<[^>]*>", "", text)  # 移除 HTML 标签
     # 使用换行符和标点符号进行初步分割，确保标点符号保留在句子末尾
-    preliminary_sentences = re.split(r'(?<=[\n。；！？\.\?!])', text)
+    preliminary_sentences = re.split(r'(?<=[\n。；！？\.\?!。])', text)
     final_sentences = []
 
     for piece in preliminary_sentences:
         if is_single_language(piece):
+            if len(piece) > 15:
+                sub_sentences = split_long_sentences(piece)
+                final_sentences.extend(sub_sentences)
             final_sentences.append(piece)
         else:
             sub_sentences = split_mixed_language(piece)
@@ -199,7 +204,7 @@ def extract_text_from_file(inputFile):
 def split_by_punctuation(sentence):
     """按照中文次级标点符号分割句子"""
     # 常见的中文次级分隔符号：逗号、分号等
-    parts = re.split(r'([，,；;])', sentence)
+    parts = re.split(r'([，,；;…、『』「」])', sentence)
     # 将标点符号与前面的词语合并，避免单独标点符号成为一个部分
     merged_parts = []
     for part in parts:
@@ -211,29 +216,13 @@ def split_by_punctuation(sentence):
 
 def split_long_sentences(sentence, max_length=30):
     """如果中文句子太长，先按标点分割，必要时使用jieba进行分词并分割"""
-    if len(sentence) > max_length and is_chinese(sentence):
-        # 首先尝试按照次级标点符号分割
-        preliminary_parts = split_by_punctuation(sentence)
-        new_sentences = []
-
-        for part in preliminary_parts:
-            # 如果部分仍然太长，使用jieba进行分词
-            if len(part) > max_length:
-                words = jieba.lcut(part)
-                current_sentence = ""
-                for word in words:
-                    if len(current_sentence) + len(word) > max_length:
-                        new_sentences.append(current_sentence)
-                        current_sentence = word
-                    else:
-                        current_sentence += word
-                if current_sentence:
-                    new_sentences.append(current_sentence)
-            else:
-                new_sentences.append(part)
 
-        return new_sentences
-    return [sentence]  # 如果句子不长或不是中文，直接返回
+    preliminary_parts = split_by_punctuation(sentence)
+    new_sentences = []
+
+    for part in preliminary_parts:
+        new_sentences.append(part)
+    return new_sentences
 
 def extract_and_convert(text):