Upload 10 files

speaker/data.py

@@ -0,0 +1,109 @@
+import torch
+import torchaudio.datasets as datasets
+import torchaudio.transforms as transforms
+from collections import defaultdict
+import random
+import layers
+
+import warnings
+
+class SpeakerMelLoader(torch.utils.data.Dataset):
+    """
+    computes mel-spectrograms from audio file and pulls the speaker ID from the
+    dataset
+    """
+
+    def __init__(self, dataset, format='speaker', speaker_utterances=4, mel_length = 128, mel_type = 'Tacotron'):
+        self.dataset = dataset
+        self.set_format(format)
+        self.speaker_utterances = speaker_utterances
+        self.mel_length = mel_length
+        self.mel_type = mel_type
+        self.mel_generators = dict()
+
+    def set_format(self,format):
+        self.format = format
+
+        if format == 'speaker':
+            self.create_speaker_index()
+
+    def create_speaker_index(self):
+        vals = [x.split('-',1) for x in self.dataset._walker]
+        speaker_map = defaultdict(list)
+
+        for i,v in enumerate(vals):
+            speaker_map[v[0]].append(i)
+        
+        self.speaker_map = speaker_map
+        self.speaker_keys = list(speaker_map.keys())
+
+    def apply_mel_gen(self, waveform, sampling_rate, channels=80):
+        if (sampling_rate, channels) not in self.mel_generators:
+            if self.mel_type == 'MFCC':
+                mel_gen = transforms.MFCC(sample_rate=sampling_rate, n_mfcc=channels)
+            elif self.mel_type == 'Mel':
+                mel_gen = transforms.MelSpectrogram(sample_rate=sampling_rate, n_mels=channels)
+            elif self.mel_type == 'Tacotron':
+                mel_gen = layers.TacotronSTFT(sampling_rate=sampling_rate,n_mel_channels=channels)
+            else:
+                raise NotImplementedError('Unsupported mel_type in MelSpeakerLoader: '+self.mel_type)
+            self.mel_generators[(sampling_rate,channels)] = mel_gen
+        else:
+            mel_gen = self.mel_generators[(sampling_rate, channels)]
+
+        if self.mel_type == 'Tacotron':
+            #Replicating from Tacotron2 data loader 
+            max_wav_value=32768.0
+            #skip normalization from Tacotron2, LibriSpeech data looks pre-normalized (all vals between 0-1)
+            audio_norm = waveform #/ max_wav_value
+            audio_norm = torch.autograd.Variable(audio_norm, requires_grad=False)
+            melspec = mel_gen.mel_spectrogram(audio_norm)
+        else:
+            audio = waveform.unsqueeze(0)
+            audio = torch.autograd.Variable(audio, requires_grad=False)
+            melspec = mel_gen(audio)
+        
+        return melspec
+
+    def get_mel(self, waveform, sampling_rate, channels=80):
+        # We previously identified that these warnings were ok.
+        with warnings.catch_warnings():
+            warnings.filterwarnings('ignore', message=r'At least one mel filterbank has all zero values.*', module=r'torchaudio.*')
+            melspec = self.apply_mel_gen(waveform, sampling_rate, channels)
+            # melspec is (1,1,channels, time) by default
+            # return (time, channels)
+            melspec = torch.squeeze(melspec).T
+            return melspec
+
+    def __getitem__(self, index):
+        if self.format == 'utterance':
+            (waveform, sample_rate, _, speaker_id, _, _) = self.dataset[index]
+            mel = self.get_mel(waveform, sample_rate)
+            return (speaker_id, mel)
+        elif self.format == 'speaker':
+            speaker_id = self.speaker_keys[index]
+            utter_indexes = random.sample(self.speaker_map[speaker_id], self.speaker_utterances)
+            mels = []
+            for i in utter_indexes:
+                (waveform, sample_rate, _, speaker_id, _, _) = self.dataset[i]
+                mel = self.get_mel(waveform, sample_rate)
+                if mel.shape[0] < self.mel_length:
+                    #Zero pad mel on the right to mel_length
+                    #pad_tuple is (dn start, dn end, dn-1 start, dn-1 end, ... , d1 start, d1 end)
+                    pad_tuple = (0,0,0,self.mel_length-mel.shape[0])
+                    mel=torch.nn.functional.pad(mel,pad_tuple)
+                    mel_frame = 0
+                else:
+                    mel_frame = random.randint(0,mel.shape[0]-self.mel_length)
+                mels.append(mel[mel_frame:mel_frame+self.mel_length,:])
+            return (speaker_id, torch.stack(mels,0))
+        else:
+            raise NotImplementedError()
+
+    def __len__(self):
+        if self.format == 'utterance':
+            return len(self.dataset)
+        elif self.format == 'speaker':
+            return len(self.speaker_keys)
+        else:
+            raise NotImplementedError()

speaker/model.py

@@ -0,0 +1,191 @@
+from torch import nn
+import numpy as np
+import torch
+from torch.nn.utils import clip_grad_norm_
+
+class SpeakerEncoder(nn.Module):
+    """ Learn speaker representation from speech utterance of arbitrary lengths.
+    """
+    def __init__(self, device, loss_device):
+        super().__init__()
+        self.loss_device = loss_device
+
+        # lstm block consisting of 3 layers
+        # takes input 80 channel log-mel spectrograms, projected to 256 dimensions
+        self.lstm = nn.LSTM(
+            input_size=80,
+            hidden_size=256,
+            num_layers=3,
+            batch_first=True,
+            dropout=0,
+            bidirectional=False
+        ).to(device)
+
+        self.linear = nn.Linear(in_features=256, out_features=256).to(device)
+        self.relu = nn.ReLU().to(device)
+        # epsilon term for numerical stability ( ie - division by 0)
+        self.epsilon = 1e-5
+
+        #Cosine similarity weights
+        self.sim_weight = nn.Parameter(torch.tensor([5.])).to(loss_device)
+        self.sim_bias = nn.Parameter(torch.tensor([-1.])).to(loss_device)
+
+    def forward(self, utterances, h_init=None, c_init=None):
+        # implement section 2.1 from https://arxiv.org/pdf/1806.04558.pdf
+        if h_init is None or c_init is None:
+            out, (hidden, cell) = self.lstm(utterances)
+        else:
+            out, (hidden, cell) = self.lstm(utterances, (h_init, c_init))
+
+        # compute speaker embedding from hidden state of final layer
+        final_hidden = hidden[-1]
+        speaker_embedding = self.relu(self.linear(final_hidden))
+
+        # l2 norm of speaker embedding
+        speaker_embedding = speaker_embedding / (torch.norm(speaker_embedding, dim=1, keepdim=True) + self.epsilon)
+        return speaker_embedding
+
+    def gradient_clipping(self):
+        self.sim_weight.grad *= 0.01
+        self.sim_bias.grad *= 0.01
+
+        #Pytorch to clip gradients if norm greater than max
+        clip_grad_norm_(self.parameters(),max_norm=3,norm_type=2)
+
+    def similarity_matrix(self, embeds, debug=False):
+        # calculate s_ji,k from section 2.1 of GE2E paper
+        # output matrix is cosine similarity between each utterance x centroid of each speaker
+        # embeds input size: (speakers, utterances, embedding size)
+
+        # Speaker centroids
+        # Equal to average of utterance embeddings for the speaker
+        # Used for neg examples (utterance comparing to false speaker)
+        # Equation 1 in paper
+        # size: (speakers, 1, embedding size)
+        speaker_centroid = torch.mean(embeds,dim=1,keepdim=True)
+
+        # Utterance exclusive centroids
+        # Equal to average of utterance embeddings for the speaker, excluding ith utterance
+        # Used for pos samples (utterance comparing to true speaker; speaker centroid exludes the utterance)
+        # Equation 8 in paper
+        # size: (speakers, utterances, embedding size)
+        num_utterance = embeds.shape[1]
+        utter_ex_centroid = (torch.sum(embeds,dim=1,keepdim=True) - embeds) / (num_utterance-1)
+
+        if debug:
+            print("e",embeds.shape)
+            print(embeds)
+            print("sc",speaker_centroid.shape)
+            print(speaker_centroid)
+            print("uc",utter_ex_centroid.shape)
+            print(utter_ex_centroid)
+
+        # Create pos and neg masks
+        num_speaker = embeds.shape[0]
+        i = torch.eye(num_speaker, dtype=torch.int)
+        pos_mask = torch.where(i)
+        neg_mask = torch.where(1-i)
+
+        if debug:
+            print("pm",len(pos_mask),len(pos_mask[0]))
+            print(pos_mask)
+            print("nm",len(neg_mask),len(neg_mask[0]))
+            print(neg_mask)
+
+        # Compile similarity matrix
+        # size: (speakers, utterances, speakers)
+        # initial size is (speakers, speakers, utterances for easier vectorization)
+        sim_matrix = torch.zeros(num_speaker, num_speaker, num_utterance).to(self.loss_device)
+        sim_matrix[pos_mask] = nn.functional.cosine_similarity(embeds,utter_ex_centroid,dim=2)
+        sim_matrix[neg_mask] = nn.functional.cosine_similarity(embeds[neg_mask[0]],speaker_centroid[neg_mask[1]],dim=2)
+        if debug:
+            print("sm",sim_matrix.shape)
+            print("pos vals",sim_matrix[pos_mask])
+            print("neg vals",sim_matrix[neg_mask])
+            print(sim_matrix)
+        
+        sim_matrix = sim_matrix.permute(0,2,1)
+
+        if debug:
+            print("sm",sim_matrix.shape)
+            print(sim_matrix)
+            print("cos sim weight", self.sim_weight)
+            print("cos sim bias", self.sim_bias)
+
+        # Apply weight / bias
+        sim_matrix = sim_matrix * self.sim_weight + self.sim_bias
+        return sim_matrix
+
+    def softmax_loss(self, embeds):
+        """
+        computes softmax loss as defined by equ 6 in the GE2E paper
+        :param embeds: shape (speakers, utterances, embedding size)
+        :return: computed softmax loss
+        """
+        # per the GE2E paper, softmax loss as defined by equ 6
+        # performs slightly better over Text-Independent Speaker
+        # Verification tasks.
+        # ref section 2.1 of the GE2E paper
+        speaker_count = embeds.shape[0]
+
+        # speaker, utterance, speaker
+        similarities = self.similarity_matrix(embeds)
+
+        # equ 6
+        loss_matrix = -similarities[torch.arange(0, speaker_count), :, torch.arange(0, speaker_count)] + \
+                      torch.log(torch.sum(torch.exp(similarities), dim=2))
+
+        # equ 10
+        return torch.sum(loss_matrix)
+
+    def contrast_loss(self, embeds):
+        """
+        computes contrast loss as defined by equ 7 in the GE2E paper
+        :param embeds: shape (speakers, utterances, embedding size)
+        :return: computed softmax loss
+        """
+        # per the GE2E paper, contrast loss as defined by equ 7
+        # performs slightly better over Text-Dependent Speaker
+        # Verification tasks.
+        # ref section 2.1 of the GE2E paper
+        speaker_count, utterance_count = embeds.shape[0:2]
+
+        # speaker, utterance, speaker
+        similarities = self.similarity_matrix(embeds)
+
+        # Janky indexing to resolve k != j
+        mask = torch.ones(similarities.shape, dtype=torch.bool)
+        mask[torch.arange(speaker_count), :, torch.arange(speaker_count)] = False
+        closest_neighbors, _ = torch.max(similarities[mask].reshape(speaker_count, utterance_count, speaker_count - 1), dim=2)
+
+        # Positive influence over matching embeddings
+        matching_embedding = similarities[torch.arange(0, speaker_count), :, torch.arange(0, speaker_count)]
+
+        # equ 7
+        loss_matrix = 1 - torch.sigmoid(matching_embedding) + torch.sigmoid(closest_neighbors)
+
+        # equ 10
+        return torch.sum(loss_matrix)
+
+    def accuracy(self, embeds):
+        """
+        computes argmax accuracy
+        :param embeds: shape (speakers, utterances, speakers)
+        :return: accuracy
+        """
+        num_speaker, num_utter = embeds.shape[:2]
+
+        similarities = self.similarity_matrix(embeds)
+        preds = torch.argmax(similarities, dim=2)
+        preds_one_hot = torch.nn.functional.one_hot(preds,num_classes = num_speaker)
+
+        actual = torch.arange(num_speaker).unsqueeze(1).repeat(1,num_utter)
+        actual_one_hot = torch.nn.functional.one_hot(actual,num_classes=num_speaker)
+
+        return torch.sum(preds_one_hot * actual_one_hot)/(num_speaker*num_utter)
+
+
+
+
+
+

speaker/preprocess.py

@@ -0,0 +1 @@
+# Reference https://github.com/CorentinJ/Real-Time-Voice-Cloning/blob/0713f860a3dd41afb56e83cff84dbdf589d5e11a/encoder/preprocess.py#L16

speaker/saved_model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6ccc0abcd0fb77104be73e6675454a06e7797bf1d4a1177181c32b648e9d75a9
+size 5697243

speaker/saved_model_e175.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52ba80266b9f45fc3d825942aae40858eeaaa73994ba86e9ed017a533dc13323
+size 5861083

speaker/tacotron_mel_e10.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9799bc6035aa1e555968c1fb2f1ca8b8bb0cdb10f11875cb4cbc1411d811a59b
+size 5861083

speaker/train.py

@@ -0,0 +1,329 @@
+import torch
+import torchaudio.datasets as datasets
+import torchaudio.transforms as transforms
+from speaker.data import SpeakerMelLoader
+from speaker.model import SpeakerEncoder
+from speaker.utils import get_mapping_array
+
+from sklearn.manifold import TSNE
+from sklearn.decomposition import PCA
+from sklearn.metrics import silhouette_score
+
+from matplotlib import pyplot as plt
+
+import os
+from os import path
+
+import numpy as np
+
+diagram_path = 'diagrams'
+accuracy_path = 'accuracy'
+loss_path = 'loss'
+silhouette_path = 'silhouette'
+tsne_path = 'tsne'
+
+
+def load_data(directory=".", batch_size=4, format='speaker', utter_per_speaker = 4, mel_type='Tacotron'):
+    dataset = SpeakerMelLoader(datasets.LIBRISPEECH(directory, download=True), format, utter_per_speaker,mel_type=mel_type)
+    return torch.utils.data.DataLoader(
+        dataset,
+        batch_size,
+        num_workers=4,
+        shuffle=True
+    )
+
+
+def load_validation(directory=".", batch_size=4, format='speaker', utter_per_speaker = 4, mel_type='Tacotron'):
+    dataset = SpeakerMelLoader(datasets.LIBRISPEECH(directory, "dev-clean",download=True), format, utter_per_speaker,mel_type=mel_type)
+    return torch.utils.data.DataLoader(
+        dataset,
+        batch_size,
+        num_workers=4,
+        shuffle=True
+    )
+
+
+def train(speaker_per_batch=4, utter_per_speaker=4, epochs=2, learning_rate=1e-4, mel_type='Tacotron'):
+    # Init data loader
+    train_loader = load_data(".", speaker_per_batch, 'speaker', utter_per_speaker,mel_type=mel_type)
+    valid_loader = load_validation(".", speaker_per_batch, 'speaker', utter_per_speaker,mel_type=mel_type)
+
+    # Device
+    # Loss calc may run faster on cpu
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    loss_device = torch.device("cpu")
+
+    # Init model
+    model = SpeakerEncoder(device, loss_device)
+    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
+
+    sil_scores = np.zeros(0)
+    gender_scores = np.zeros(0)
+    val_losses = np.zeros(0)
+    val_accuracy = np.zeros(0)
+
+    gender_mapper = get_mapping_array()
+
+    # Train loop
+    for e in range(epochs):
+        print('epoch:', e+1, 'of', epochs)
+
+        model.train()
+        # train_ids = np.zeros(0)
+        # train_embeds = np.zeros((0, 256))
+        for step, batch in enumerate(train_loader):
+            #Forward
+            #inputs: (speaker, utter, mel_len, mel_channel)
+            speaker_id, inputs = batch
+            #embed_inputs: (speaker*utter, mel_len, mel_channel)
+            embed_inputs = inputs.reshape(-1, *(inputs.shape[2:])).to(device)
+            #embeds: (speaker*utter, embed_dim)
+            embeds = model(embed_inputs)
+            #loss_embeds: (speaker, utter, embed_dim)
+            loss_embeds = embeds.view((speaker_per_batch,utter_per_speaker,-1)).to(loss_device)
+            loss = model.softmax_loss(loss_embeds)
+
+            if step % 10 == 0:
+                print('train e{}-s{}:'.format(e + 1, step + 1), 'loss', loss)
+
+            #Backward
+            model.zero_grad()
+            loss.backward()
+            model.gradient_clipping()
+            optimizer.step()
+
+            # train_ids = np.concatenate((train_ids, np.repeat(speaker_id, inputs.shape[1])))
+            # train_embeds = np.concatenate((train_embeds, embeds))
+        
+        model.eval()
+        loss = 0
+        acc = 0
+
+        valid_ids = np.zeros(0)
+        valid_embeds = np.zeros((0, 256))
+
+        for step,batch in enumerate(valid_loader):
+            with torch.no_grad():
+                speaker_id, inputs = batch
+                embed_inputs = inputs.reshape(-1, *(inputs.shape[2:])).to(device)
+                embeds = model(embed_inputs)
+                loss_embeds = embeds.view((speaker_per_batch,utter_per_speaker,-1)).to(loss_device)
+                loss += model.softmax_loss(loss_embeds)
+                acc += model.accuracy(loss_embeds)
+                valid_ids = np.concatenate((valid_ids, np.repeat(speaker_id, inputs.shape[1])))
+                valid_embeds = np.concatenate((valid_embeds, embeds.to(loss_device).detach()))
+
+        val_losses = np.concatenate((val_losses, [loss.to(loss_device).detach() / (step + 1)]))
+        val_accuracy = np.concatenate((val_accuracy, [acc.to(loss_device).detach() / (step + 1)]))
+        sil_scores = np.concatenate((sil_scores, [silhouette_score(valid_embeds, valid_ids)]))
+        gender_scores = np.concatenate((gender_scores, [silhouette_score(valid_embeds, gender_mapper[valid_ids.astype('int')])]))
+        print('valid e{}'.format(e + 1), 'loss', val_losses[-1])
+        print('valid e{}'.format(e + 1), 'accuracy', val_accuracy[-1])
+        print('silhouette score', sil_scores[-1])
+        print('gender silhouette score', gender_scores[-1])
+
+        plot_speaker_embeddings(valid_embeds, valid_ids, f'tsne_e{e + 1}_speaker.png', f'T-SNE Plot: Epoch {e + 1}')
+        plot_random_embeddings(valid_embeds, valid_ids, f'tsne_e{e + 1}_random.png', title=f'T-SNE Plot: Epoch {e + 1}')
+        plot_gender_embeddings(valid_embeds, valid_ids, f'tsne_e{e + 1}_gender.png', f'T-SNE Plot: Epoch {e + 1}')
+
+        save_model(model, path.join('speaker', f'saved_model_e{e + 1}.pt'))
+
+        plt.figure()
+        plt.title('Silhouette Scores')
+        plt.xlabel('Epoch')
+        plt.ylabel('Silhouette Score')
+        plt.plot(np.arange(e + 1) + 1, sil_scores)
+        # plt.show()
+        plt.savefig(path.join(diagram_path, silhouette_path, f'sil_scores_{e + 1}.png'))
+        plt.close()
+
+        plt.figure()
+        plt.title('Silhouette Scores over Gender')
+        plt.xlabel('Epoch')
+        plt.ylabel('Silhouette Score')
+        plt.plot(np.arange(e + 1) + 1, gender_scores)
+        # plt.show()
+        plt.savefig(path.join(diagram_path, silhouette_path, f'gender_scores_{e + 1}.png'))
+        plt.close()
+
+        plt.figure()
+        plt.title('Validation Loss')
+        plt.xlabel('Epoch')
+        plt.ylabel('Loss')
+        plt.plot(np.arange(e + 1) + 1, val_losses)
+        # plt.show()
+        plt.savefig(path.join(diagram_path, loss_path, f'val_losses_{e + 1}.png'))
+        plt.close()
+
+        plt.figure()
+        plt.title('Validation Accuracy')
+        plt.xlabel('Epoch')
+        plt.ylabel('Accuracy')
+        plt.plot(np.arange(e + 1) + 1, val_accuracy)
+        # plt.show()
+        plt.savefig(path.join(diagram_path, accuracy_path, f'val_accuracy_{e + 1}.png'))
+        plt.close()
+        
+    return model
+
+
+def save_model(model, path):
+    #Save model state to path
+    torch.save(model.state_dict(),path)
+
+
+def load_model(path, device = None):
+    #Instantiate Model
+    if device is None:
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    loss_device = torch.device("cpu")
+    model = SpeakerEncoder(device, loss_device)
+
+    #Load model state
+    model.load_state_dict(torch.load(path))
+    # Try this if running on multi-gpu setup or running model on cpu
+    # https://pytorch.org/tutorials/beginner/saving_loading_models.html#saving-loading-model-across-devices
+    # model.load_state_dict(torch.load(PATH, map_location=device))
+    return model
+
+
+def check_model(path):
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    loss_device = torch.device("cpu")
+
+    print('**loading model')
+    model = load_model(path)
+
+    print('**loading data')
+    # data = load_data()
+    data = load_validation()
+
+    print('**running model')
+    loss_total = 0
+    acc_total = 0
+    all_ids = np.zeros(0)
+    all_embeds = np.zeros((0, 256))
+
+    for step, batch in enumerate(data):
+        speaker_id, inputs = batch
+
+        print('batch:', step)
+        embed_inputs = inputs.reshape(-1, *(inputs.shape[2:])).to(device)
+        embeds = model(embed_inputs)
+        loss_embeds = embeds.view(*(inputs.shape[:2]),-1).to(loss_device)
+        loss = model.softmax_loss(loss_embeds)
+        accuracy = model.accuracy(loss_embeds)
+
+        all_ids = np.concatenate((all_ids, np.repeat(speaker_id, inputs.shape[1])))
+        all_embeds = np.concatenate((all_embeds, embeds.to(loss_device).detach()))
+
+        loss_total += loss
+        acc_total += accuracy
+        
+        # print('inputs.shape',inputs.shape)
+        # print('embed_inputs.embed_inputs',embeds.shape)
+        # print('embeds.shape',embeds.shape)
+        # print('loss_embeds.shape',loss_embeds.shape)
+        # print('loss.shape',loss.shape)
+        # print('loss',loss)
+        # print('accuracy',accuracy)
+    
+    print('average loss', loss_total / (step+1))
+    print('average accuracy', acc_total / (step+1))
+    print('silhouette score', silhouette_score(all_embeds, all_ids))
+    plot_speaker_embeddings(all_embeds, all_ids, f'tsne_saved_speaker.png', f'T-SNE Plot')
+    plot_random_embeddings(all_embeds, all_ids, f'tsne_saved_random.png', title=f'T-SNE Plot')
+    plot_gender_embeddings(all_embeds, all_ids, f'tsne_saved_gender.png', f'T-SNE Plot')
+
+
+def plot_gender_embeddings(embeddings, ids, filename, title='T-SNE Plot'):
+    # Per https://scikit-learn.org/stable/modules/generated/sklearn.manifold.TSNE.html
+    # reducing dimensionality before running TSNE
+    pca = PCA(50)
+    reduction = pca.fit_transform(embeddings)
+    tsne = TSNE(init='pca', learning_rate='auto')
+    transformed = tsne.fit_transform(reduction)
+
+    gender_mapper = get_mapping_array()
+    genders = gender_mapper[ids.astype('int')]
+    females = genders == 1
+    males = genders == 2
+
+    plt.figure()
+    plt.title(title)
+
+    plt.scatter(transformed[females, 0], transformed[females, 1], label='Female')
+    plt.scatter(transformed[males, 0], transformed[males, 1], label='Male')
+    plt.legend()
+    plt.grid()
+    # plt.show()
+    plt.savefig(path.join(diagram_path, tsne_path, filename))
+    plt.close()
+
+
+def plot_speaker_embeddings(embeddings, ids, filename, title='T-SNE Plot'):
+    # Per https://scikit-learn.org/stable/modules/generated/sklearn.manifold.TSNE.html
+    # reducing dimensionality before running TSNE
+    pca = PCA(50)
+    reduction = pca.fit_transform(embeddings)
+    tsne = TSNE(init='pca', learning_rate='auto')
+    transformed = tsne.fit_transform(reduction)
+
+    ids = ids.astype('int')
+    unique_ids = np.unique(ids)
+
+    plt.figure()
+    plt.title(f'{title} Speakers')
+
+    for speaker_id in unique_ids:
+        speaker_idx = ids == speaker_id
+        plt.scatter(transformed[speaker_idx, 0], transformed[speaker_idx, 1], label=f'Speaker {speaker_id}')
+
+    # plt.legend()
+    plt.grid()
+    # plt.show()
+    plt.savefig(path.join(diagram_path, tsne_path, filename))
+    plt.close()
+
+
+def plot_random_embeddings(embeddings, ids, filename, size=15, title='T-SNE Plot Random'):
+    # Per https://scikit-learn.org/stable/modules/generated/sklearn.manifold.TSNE.html
+    # reducing dimensionality before running TSNE
+    pca = PCA(50)
+    reduction = pca.fit_transform(embeddings)
+    tsne = TSNE(init='pca', learning_rate='auto')
+    transformed = tsne.fit_transform(reduction)
+
+    ids = ids.astype('int')
+    unique_ids = np.unique(ids)
+    random_unique_ids = np.random.choice(ids, size=min(size, unique_ids.size), replace=False)
+
+    plt.figure()
+
+    plt.title(f'{title} - {random_unique_ids.size} Speakers')
+
+    for speaker_id in random_unique_ids:
+        speaker_idx = ids == speaker_id
+        plt.scatter(transformed[speaker_idx, 0], transformed[speaker_idx, 1], label=f'Speaker {speaker_id}')
+
+    # plt.legend()
+    plt.grid()
+    # plt.show()
+    plt.savefig(path.join(diagram_path, tsne_path, filename))
+    plt.close()
+
+
+if __name__ == '__main__':
+    os.makedirs(diagram_path, exist_ok=True)
+    os.makedirs(path.join(diagram_path, loss_path), exist_ok=True)
+    os.makedirs(path.join(diagram_path, accuracy_path), exist_ok=True)
+    os.makedirs(path.join(diagram_path, tsne_path), exist_ok=True)
+    os.makedirs(path.join(diagram_path, silhouette_path), exist_ok=True)
+    # for speaker_id, mel in load_data():
+    #     print(speaker_id, mel.shape)
+
+    # Might make sense to adjust speaker / utterance per batch, e.g. 64/10    
+    m = train(epochs=300)
+
+    # save_model(m,'speaker/saved_model.pt')
+    check_model('speaker/saved_model_e175.pt')

speaker/utils.py

@@ -0,0 +1,28 @@
+
+import numpy as np
+
+__mapping_array = None
+
+with open('speaker/speakers.txt') as speakers:
+    lines = []
+    for line in speakers.readlines():
+        if line[0] == ';':
+            continue
+        lines.append(line)
+
+    rows = [line.split('|') for line in lines]
+
+    __mapping_list = [(int(row[0].strip()), row[1].strip()) for row in rows]
+
+    max_id = max([speaker_id for (speaker_id, _) in __mapping_list])\
+
+    __mapping_array = np.zeros(max_id + 1,)
+    for speaker_id, gender in __mapping_list:
+        if gender == 'F':
+            __mapping_array[speaker_id] = 1
+        else:
+            __mapping_array[speaker_id] = 2
+
+
+def get_mapping_array():
+    return np.copy(__mapping_array)