Update GPT-SoVITS

gpt_sovits/AR/models/t2s_model.py

@@ -506,18 +506,32 @@ class Text2SemanticDecoder(nn.Module):
 
     def infer_panel_batch_infer_with_flash_attn(
             self,
-            x,  #####全部文本token
-            x_lens,
-            prompts,  ####参考音频token
-            bert_feature,
+            x: List[torch.LongTensor],  #####全部文本token
+            x_lens: torch.LongTensor,
+            prompts: torch.LongTensor,  ####参考音频token
+            bert_feature: List[torch.LongTensor],
             top_k: int = -100,
             top_p: float = 100,
             early_stop_num: int = -1,
             temperature: float = 1.0,
     ):
-
-        bert_feature = self.bert_proj(bert_feature.transpose(1, 2))
-        x = self.ar_text_embedding(x)
+        # 先对phones进行embedding、对bert_features进行project，再pad到相同长度，以缓解复读问题。（可能还有其他因素导致复读）
+        max_len = 0
+        for x_item, bert_item in zip(x, bert_feature):
+            max_len = max(max_len, x_item.shape[0], bert_item.shape[1])
+        x_list = [self.ar_text_embedding(item) for item in x]
+        x_list = [F.pad(item, (0, 0, 0, max_len - item.shape[0]), value=0) if item.shape[0] < max_len else item for item
+                  in x_list]
+        x = torch.stack(x_list, dim=0)
+
+        bert_features_list = [self.bert_proj(item.transpose(0, 1)) for item in bert_feature]
+        bert_features_list = [
+            F.pad(item, (0, 0, 0, max_len - item.shape[0]), value=0) if item.shape[0] < max_len else item for item in
+            bert_features_list]
+        bert_feature = torch.stack(bert_features_list, dim=0)
+
+        # bert_feature = self.bert_proj(bert_feature.transpose(1, 2))
+        # x = self.ar_text_embedding(x)
         x = x + bert_feature
         x = self.ar_text_position(x)
 
@@ -660,17 +674,33 @@ class Text2SemanticDecoder(nn.Module):
 
     def infer_panel_batch_only(
             self,
-            x,  #####全部文本token
-            x_lens,
-            prompts,  ####参考音频token
-            bert_feature,
+            x: List[torch.LongTensor],  #####全部文本token
+            x_lens: torch.LongTensor,
+            prompts: torch.LongTensor,  ####参考音频token
+            bert_feature: List[torch.LongTensor],
             top_k: int = -100,
-            top_p: float = 100,
+            top_p: int = 100,
             early_stop_num: int = -1,
             temperature: float = 1.0,
     ):
-        x = self.ar_text_embedding(x)
-        x = x + self.bert_proj(bert_feature.transpose(1, 2))
+        # 先对phones进行embedding、对bert_features进行project，再pad到相同长度，以缓解复读问题。（可能还有其他因素导致复读）
+        max_len = 0
+        for x_item, bert_item in zip(x, bert_feature):
+            max_len = max(max_len, x_item.shape[0], bert_item.shape[1])
+        x_list = [self.ar_text_embedding(item) for item in x]
+        x_list = [F.pad(item, (0, 0, 0, max_len - item.shape[0]), value=0) if item.shape[0] < max_len else item for item
+                  in x_list]
+        x = torch.stack(x_list, dim=0)
+
+        bert_features_list = [self.bert_proj(item.transpose(0, 1)) for item in bert_feature]
+        bert_features_list = [
+            F.pad(item, (0, 0, 0, max_len - item.shape[0]), value=0) if item.shape[0] < max_len else item for item in
+            bert_features_list]
+        bert_feature = torch.stack(bert_features_list, dim=0)
+
+        # bert_feature = self.bert_proj(bert_feature.transpose(1, 2))
+        # x = self.ar_text_embedding(x)
+        x = x + bert_feature
         x = self.ar_text_position(x)
 
         # AR Decoder
@@ -709,19 +739,35 @@ class Text2SemanticDecoder(nn.Module):
             y = torch.zeros(x.shape[0], 0, dtype=torch.int, device=x.device)
             ref_free = True
 
-        x_attn_mask_pad = F.pad(
+        ##### create mask #####
+        bsz = x.shape[0]
+        src_len = x_len + y_len
+        y_lens = torch.LongTensor([y_len] * bsz).to(x.device)
+        y_mask = make_pad_mask(y_lens)
+        x_mask = make_pad_mask(x_lens)
+
+        # (bsz, x_len + y_len)
+        xy_padding_mask = torch.concat([x_mask, y_mask], dim=1)
+
+        x_mask = F.pad(
             x_attn_mask,
             (0, y_len),  ###xx的纯0扩展到xx纯0+xy纯1，(x,x+y)
             value=True,
         )
-        y_attn_mask = F.pad(  ###yy的右上1扩展到左边xy的0,(y,x+y)
+        y_mask = F.pad(  ###yy的右上1扩展到左边xy的0,(y,x+y)
             torch.triu(torch.ones(y_len, y_len, dtype=torch.bool), diagonal=1),
             (x_len, 0),
             value=False,
         )
-        xy_attn_mask = torch.concat([x_attn_mask_pad, y_attn_mask], dim=0).to(
-            x.device
-        )
+
+        xy_mask = torch.concat([x_mask, y_mask], dim=0).view(1, src_len, src_len).expand(bsz * self.num_head, -1,
+                                                                                         -1).to(x.device)
+        # xy_mask = torch.triu(torch.ones(src_len, src_len, dtype=torch.bool, device=x.device), diagonal=1)
+        xy_padding_mask = xy_padding_mask.view(bsz, 1, src_len).expand(bsz, src_len, src_len).repeat(self.num_head, 1,
+                                                                                                     1)
+        xy_attn_mask = xy_mask.logical_or(xy_padding_mask)
+        new_attn_mask = torch.zeros_like(xy_attn_mask, dtype=x.dtype)
+        xy_attn_mask = new_attn_mask.masked_fill(xy_attn_mask, float("-inf"))
 
         y_list = [None] * y.shape[0]
         batch_idx_map = list(range(y.shape[0]))
@@ -817,4 +863,4 @@ class Text2SemanticDecoder(nn.Module):
 
         if ref_free:
             return y_list, [0] * x.shape[0]
-        return y_list, idx_list
+        return y_list, idx_list

gpt_sovits/text/tone_sandhi.py

@@ -663,32 +663,29 @@ class ToneSandhi:
     # input seg: [('听', 'v'), ('一', 'm'), ('听', 'v')]
     # output seg: [['听一听', 'v']]
     def _merge_yi(self, seg: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
-        new_seg = [] * len(seg)
+        new_seg = []
         # function 1
-        i = 0
-        while i < len(seg):
-            word, pos = seg[i]
+        for i, (word, pos) in enumerate(seg):
             if (
-                    i - 1 >= 0
-                    and word == "一"
-                    and i + 1 < len(seg)
-                    and seg[i - 1][0] == seg[i + 1][0]
-                    and seg[i - 1][1] == "v"
+                i - 1 >= 0
+                and word == "一"
+                and i + 1 < len(seg)
+                and seg[i - 1][0] == seg[i + 1][0]
+                and seg[i - 1][1] == "v"
+                and seg[i + 1][1] == "v"
             ):
                 new_seg[i - 1][0] = new_seg[i - 1][0] + "一" + new_seg[i - 1][0]
-                i += 2
             else:
                 if (
-                        i - 2 >= 0
-                        and seg[i - 1][0] == "一"
-                        and seg[i - 2][0] == word
-                        and pos == "v"
+                    i - 2 >= 0
+                    and seg[i - 1][0] == "一"
+                    and seg[i - 2][0] == word
+                    and pos == "v"
                 ):
                     continue
                 else:
                     new_seg.append([word, pos])
-                i += 1
-        seg = [i for i in new_seg if len(i) > 0]
+        seg = new_seg
         new_seg = []
         # function 2
         for i, (word, pos) in enumerate(seg):

gpt_sovits/text/zh_normalization/num.py

@@ -172,7 +172,23 @@ def replace_range(match) -> str:
     return result
 
 
-def _get_value(value_string: str, use_zero: bool=True) -> List[str]:
+# ~至表达式
+RE_TO_RANGE = re.compile(
+    r'((-?)((\d+)(\.\d+)?)|(\.(\d+)))(%|°C|℃|度|摄氏度|cm2|cm²|cm3|cm³|cm|db|ds|kg|km|m2|m²|m³|m3|ml|m|mm|s)[~]((-?)((\d+)(\.\d+)?)|(\.(\d+)))(%|°C|℃|度|摄氏度|cm2|cm²|cm3|cm³|cm|db|ds|kg|km|m2|m²|m³|m3|ml|m|mm|s)')
+
+
+def replace_to_range(match) -> str:
+    """
+    Args:
+        match (re.Match)
+    Returns:
+        str
+    """
+    result = match.group(0).replace('~', '至')
+    return result
+
+
+def _get_value(value_string: str, use_zero: bool = True) -> List[str]:
     stripped = value_string.lstrip('0')
     if len(stripped) == 0:
         return []
@@ -202,7 +218,7 @@ def verbalize_cardinal(value_string: str) -> str:
     result_symbols = _get_value(value_string)
     # verbalized number starting with '一十*' is abbreviated as `十*`
     if len(result_symbols) >= 2 and result_symbols[0] == DIGITS[
-            '1'] and result_symbols[1] == UNITS[1]:
+        '1'] and result_symbols[1] == UNITS[1]:
         result_symbols = result_symbols[1:]
     return ''.join(result_symbols)
 

gpt_sovits/text/zh_normalization/text_normlization.py

@@ -33,6 +33,7 @@ from .num import RE_NUMBER
 from .num import RE_PERCENTAGE
 from .num import RE_POSITIVE_QUANTIFIERS
 from .num import RE_RANGE
+from .num import RE_TO_RANGE
 from .num import replace_default_num
 from .num import replace_frac
 from .num import replace_negative_num
@@ -40,6 +41,7 @@ from .num import replace_number
 from .num import replace_percentage
 from .num import replace_positive_quantifier
 from .num import replace_range
+from .num import replace_to_range
 from .phonecode import RE_MOBILE_PHONE
 from .phonecode import RE_NATIONAL_UNIFORM_NUMBER
 from .phonecode import RE_TELEPHONE
@@ -65,7 +67,7 @@ class TextNormalizer():
         if lang == "zh":
             text = text.replace(" ", "")
             # 过滤掉特殊字符
-            text = re.sub(r'[——《》【】<=>{}()（）#&@“”^_|…\\]', '', text)
+            text = re.sub(r'[——《》【】<=>{}()（）#&@“”^_|\\]', '', text)
         text = self.SENTENCE_SPLITOR.sub(r'\1\n', text)
         text = text.strip()
         sentences = [sentence.strip() for sentence in re.split(r'\n+', text)]
@@ -73,8 +75,8 @@ class TextNormalizer():
 
     def _post_replace(self, sentence: str) -> str:
         sentence = sentence.replace('/', '每')
-        sentence = sentence.replace('~', '至')
-        sentence = sentence.replace('～', '至')
+        # sentence = sentence.replace('~', '至')
+        # sentence = sentence.replace('～', '至')
         sentence = sentence.replace('①', '一')
         sentence = sentence.replace('②', '二')
         sentence = sentence.replace('③', '三')
@@ -111,7 +113,7 @@ class TextNormalizer():
         sentence = sentence.replace('ψ', '普赛').replace('Ψ', '普赛')
         sentence = sentence.replace('ω', '欧米伽').replace('Ω', '欧米伽')
         # re filter special characters, have one more character "-" than line 68
-        sentence = re.sub(r'[-——《》【】<=>{}()（）#&@“”^_|…\\]', '', sentence)
+        sentence = re.sub(r'[-——《》【】<=>{}()（）#&@“”^_|\\]', '', sentence)
         return sentence
 
     def normalize_sentence(self, sentence: str) -> str:
@@ -128,6 +130,8 @@ class TextNormalizer():
         sentence = RE_TIME_RANGE.sub(replace_time, sentence)
         sentence = RE_TIME.sub(replace_time, sentence)
 
+        # 处理~波浪号作为至的替换
+        sentence = RE_TO_RANGE.sub(replace_to_range, sentence)
         sentence = RE_TEMPERATURE.sub(replace_temperature, sentence)
         sentence = replace_measure(sentence)
         sentence = RE_FRAC.sub(replace_frac, sentence)