del fixed embedding voice diffussion

Modules/diffusion/modules.py

@@ -146,6 +146,7 @@ class StyleTransformer1d(nn.Module):
         return mapping
             
     def run(self, x, time, embedding, features):
+        # called by forward()
         
         mapping = self.get_mapping(time, features)
         x = torch.cat([x.expand(-1, embedding.size(1), -1), embedding], axis=-1)
@@ -161,37 +162,22 @@ class StyleTransformer1d(nn.Module):
         
         return x
         
-    def forward(self, x: Tensor, 
-                time: Tensor, 
-                embedding_mask_proba: float = 0.0,
+    def forward(self, 
+                x, 
+                time,
                 embedding= None, 
-                features = None,
-               embedding_scale: float = 1.0) -> Tensor:
-        
-        b, device = embedding.shape[0], embedding.device
-        fixed_embedding = self.fixed_embedding(embedding)
-        if embedding_mask_proba > 0.0:
-            # Randomly mask embedding
-            batch_mask = rand_bool(
-                shape=(b, 1, 1), proba=embedding_mask_proba, device=device
-            )
-            embedding = torch.where(batch_mask, fixed_embedding, embedding)
+                features = None):
 
-        if embedding_scale != 1.0:
-            
-            
-            out = self.run(x, time, embedding=embedding, features=features)
-            out_masked = self.run(x, time, embedding=fixed_embedding, features=features)
-            
-            raise ValueError
-            return out_masked + (out - out_masked) * embedding_scale
-            
-        else:
-            # raise ValueError
-            return self.run(x, time, embedding=embedding, features=features)
-          
-        
-        return x
+        b, device = embedding.shape[0], embedding.device
+        # if 
+                # embedding_mask_proba: float = 0.0, > 0
+        # fixed_embedding = self.fixed_embedding(embedding)
+        # embedding = torch.where(batch_mask, fixed_embedding, embedding)
+        return self.run(x, 
+                        time, 
+                        embedding=embedding,
+                        # embedding=self.fixed_embedding(embedding),  # fixedemb has noisy beginnings on chapters.wav
+                        features=features)
 
 
 class StyleTransformerBlock(nn.Module):
@@ -216,24 +202,11 @@ class StyleTransformerBlock(nn.Module):
             features=features,
             style_dim=style_dim,
             num_heads=num_heads,
-            head_features=head_features,
-            use_rel_pos=use_rel_pos,
-            # rel_pos_num_buckets=rel_pos_num_buckets,
-            # rel_pos_max_distance=rel_pos_max_distance,
+            head_features=head_features
         )
 
         if self.use_cross_attention:
             raise ValueError
-            # self.cross_attention = StyleAttention(
-            #     features=features,
-            #     style_dim=style_dim,
-            #     num_heads=num_heads,
-            #     head_features=head_features,
-            #     context_features=context_features,
-            #     use_rel_pos=use_rel_pos,
-            #     rel_pos_num_buckets=rel_pos_num_buckets,
-            #     rel_pos_max_distance=rel_pos_max_distance,
-            # )
 
         self.feed_forward = FeedForward(features=features, multiplier=multiplier)
 
@@ -254,7 +227,7 @@ class StyleAttention(nn.Module):
         head_features: int,
         num_heads: int,
         context_features = None,
-        use_rel_pos: bool,
+        # use_rel_pos: bool,
         # rel_pos_num_buckets: Optional[int] = None,
         # rel_pos_max_distance: Optional[int] = None,
     ):
@@ -274,23 +247,20 @@ class StyleAttention(nn.Module):
         self.attention = AttentionBase(
             features,
             num_heads=num_heads,
-            head_features=head_features,
-            use_rel_pos=use_rel_pos,
-            # rel_pos_num_buckets=rel_pos_num_buckets,
-            # rel_pos_max_distance=rel_pos_max_distance,
+            head_features=head_features
         )
 
-    def forward(self, x: Tensor, s: Tensor, *, context = None):
+    def forward(self, x, s, *, context = None):
         
-        # raise ValueError
-        # Use context if provided
+        if context is not None:
+            raise ValueError
         context = default(context, x)
-        # print(context.shape,'ppppppppppppppppppppppppppppppppppppppppppp')  # bs, time, 1024
-        # Normalize then compute q from input and k,v from context
+        
+        
         x, context = self.norm(x, s), self.norm_context(context, s)
         
         q, k, v = (self.to_q(x), *torch.chunk(self.to_kv(context), chunks=2, dim=-1))
-        # Compute and return attention
+        
         return self.attention(q, k, v)
 
 
@@ -310,25 +280,13 @@ class AttentionBase(nn.Module):
         features,
         *,
         head_features,
-        num_heads,
-        use_rel_pos,
-        out_features = None,
-        # rel_pos_num_buckets: Optional[int] = None,
-        # rel_pos_max_distance: Optional[int] = None,
-    ):
+        num_heads):
         super().__init__()
         self.scale = head_features ** -0.5
         self.num_heads = num_heads
-        self.use_rel_pos = use_rel_pos
-        mid_features = head_features * num_heads
-
-        if use_rel_pos:
-            raise ValueError
-
-        if out_features is None:
-            out_features = features
-            
-        self.to_out = nn.Linear(in_features=mid_features, out_features=out_features)
+        mid_features = head_features * num_heads            
+        self.to_out = nn.Linear(in_features=mid_features, 
+                                out_features=features)
 
     def forward(self, q: Tensor, k: Tensor, v: Tensor) -> Tensor:
         # Split heads
@@ -358,7 +316,7 @@ class Attention(nn.Module):
         num_heads,
         out_features=None,
         context_features=None,
-        use_rel_pos,
+        # use_rel_pos,
         # rel_pos_num_buckets: Optional[int] = None,
         # rel_pos_max_distance: Optional[int] = None,
     ):
@@ -381,7 +339,7 @@ class Attention(nn.Module):
             out_features=out_features,
             num_heads=num_heads,
             head_features=head_features,
-            use_rel_pos=use_rel_pos,
+            # use_rel_pos=use_rel_pos,
             # rel_pos_num_buckets=rel_pos_num_buckets,
             # rel_pos_max_distance=rel_pos_max_distance,
         )

Modules/diffusion/sampler.py

@@ -1,61 +1,18 @@
-from math import atan, cos, pi, sin, sqrt
+from math import sqrt
 import torch.nn as nn
 from einops import rearrange
 from torch import Tensor
-
 from functools import reduce
-from inspect import isfunction
-from math import ceil, floor, log2, pi
-# from typing import Callable, Dict, List, Optional, Sequence, Tuple, TypeVar, Union
+# from inspect import isfunction
+# from math import ceil, floor, log2, pi
 import torch
 import torch.nn.functional as F
-from einops import rearrange
-from torch import Generator, Tensor
-
-
-
-
-
-
-# def is_sequence(obj: T) -> TypeGuard[Union[list, tuple]]:
-#     return isinstance(obj, list) or isinstance(obj, tuple)
-
 
 def default(val, d):
     if val is not None: #exists(val):
         return val
     return d #d() if isfunction(d) else d
 
-
-# def to_list(val: Union[T, Sequence[T]]) -> List[T]:
-#     if isinstance(val, tuple):
-#         return list(val)
-#     if isinstance(val, list):
-#         return val
-#     return [val]  # type: ignore
-
-
-# def prod(vals: Sequence[int]) -> int:
-#     return reduce(lambda x, y: x * y, vals)
-
-
-def closest_power_2(x: float) -> int:
-    exponent = log2(x)
-    distance_fn = lambda z: abs(x - 2 ** z)  # noqa
-    exponent_closest = min((floor(exponent), ceil(exponent)), key=distance_fn)
-    return 2 ** int(exponent_closest)
-
-def rand_bool(shape, proba, device = None):
-    if proba == 1:
-        return torch.ones(shape, device=device, dtype=torch.bool)
-    elif proba == 0:
-        return torch.zeros(shape, device=device, dtype=torch.bool)
-    else:
-        return torch.bernoulli(torch.full(shape, proba, device=device)).to(torch.bool)
-
-# ============================= END functions from diffusION.utils 
-
-
 class LogNormalDistribution():
     def __init__(self, mean: float, std: float):
         self.mean = mean
@@ -238,7 +195,8 @@ class DiffusionSampler(nn.Module):
         
         # Compute sigmas using schedule
         sigmas = self.sigma_schedule(num_steps, device)
-        # Append additional kwargs to denoise function (used e.g. for conditional unet)
+        
+        # L242 KWARGS dict_keys(['embedding', 'features'])
         fn = lambda *a, **ka: self.denoise_fn(*a, **{**ka, **kwargs})  # noqa
         # Sample using sampler
         x = self.sampler(noise, fn=fn, sigmas=sigmas, num_steps=num_steps)

api.py

@@ -171,8 +171,8 @@ def tts_multi_sentence(precomputed_style_vector=None,
                         precomputed_style_vector,
                                     alpha=0.3,
                                     beta=0.7,
-                                    diffusion_steps=diffusion_steps,
-                                    embedding_scale=1))
+                                    diffusion_steps=diffusion_steps)
+                     )
         x = np.concatenate(x)
                     
     # Fallback - MMS TTS - Non-English
@@ -530,7 +530,7 @@ def serve_wav():
 
     # audios = [msinference.inference(text, 
     #                                 msinference.compute_style(f'voices/{voice}.wav'), 
-    #                                 alpha=0.3, beta=0.7, diffusion_steps=7, embedding_scale=1)]
+    #                                 alpha=0.3, beta=0.7, diffusion_steps=7)]
     # # for t in [text]:
     # output_buffer = io.BytesIO()
     # write(output_buffer, 24000, np.concatenate(audios))

models.py

@@ -69,6 +69,7 @@ class AudioDiffusionConditional(nn.Module):
 
     def forward(self, *args, **kwargs):
         default_kwargs = dict(embedding_mask_proba=self.embedding_mask_proba)
+        # here embedding_scale = 1.0 is passed to DiffusionSampler() - del no-op if scale = 1.0
         return self.diffusion(*args, **{**default_kwargs, **kwargs})
 
     # def sample(self, *args, **kwargs):

msinference.py

@@ -174,7 +174,6 @@ def inference(text,
               alpha = 0.3,
               beta = 0.7,
               diffusion_steps=7, # 7 if voice is native English else 5 for non-native
-              embedding_scale=1,
               use_gruut=False):
     text = text.strip()
     ps = global_phonemizer.phonemize([text])
@@ -213,7 +212,6 @@ def inference(text,
 
         s_pred = sampler(noise = torch.randn((1, 256)).unsqueeze(1).to(device),
                                           embedding=bert_dur,
-                                          embedding_scale=embedding_scale,
                                             features=ref_s, # reference from the same speaker as the embedding
                                              num_steps=diffusion_steps).squeeze(1)