DTLN (models)

models/DTLN (alekya)/.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

models/DTLN (alekya)/DTLN_model.py

@@ -0,0 +1,641 @@
+# -*- coding: utf-8 -*-
+"""
+This File contains everything to train the DTLN model.
+
+For running the training see "run_training.py".
+To run evaluation with the provided pretrained model see "run_evaluation.py".
+
+Author: Nils L. Westhausen (nils.westhausen@uol.de)
+Version: 24.06.2020
+
+This code is licensed under the terms of the MIT-license.
+"""
+
+
+import os, fnmatch
+import tensorflow.keras as keras
+from tensorflow.keras.models import Model
+from tensorflow.keras.layers import Activation, Dense, LSTM, Dropout, \
+    Lambda, Input, Multiply, Layer, Conv1D
+from tensorflow.keras.callbacks import ReduceLROnPlateau, CSVLogger, \
+    EarlyStopping, ModelCheckpoint
+import tensorflow as tf
+import soundfile as sf
+from wavinfo import WavInfoReader
+from random import shuffle, seed
+import numpy as np
+
+
+
+class audio_generator():
+    '''
+    Class to create a Tensorflow dataset based on an iterator from a large scale 
+    audio dataset. This audio generator only supports single channel audio files.
+    '''
+    
+    def __init__(self, path_to_input, path_to_s1, len_of_samples, fs, train_flag=False):
+        '''
+        Constructor of the audio generator class.
+        Inputs:
+            path_to_input       path to the mixtures
+            path_to_s1          path to the target source data
+            len_of_samples      length of audio snippets in samples
+            fs                  sampling rate
+            train_flag          flag for activate shuffling of files
+        '''
+        # set inputs to properties
+        self.path_to_input = path_to_input
+        self.path_to_s1 = path_to_s1
+        self.len_of_samples = len_of_samples
+        self.fs = fs
+        self.train_flag=train_flag
+        # count the number of samples in your data set (depending on your disk,
+        #                                               this can take some time)
+        self.count_samples()
+        # create iterable tf.data.Dataset object
+        self.create_tf_data_obj()
+        
+    def count_samples(self):
+        '''
+        Method to list the data of the dataset and count the number of samples. 
+        '''
+
+        # list .wav files in directory
+        self.file_names = fnmatch.filter(os.listdir(self.path_to_input), '*.wav')
+        # count the number of samples contained in the dataset
+        self.total_samples = 0
+        for file in self.file_names:
+            info = WavInfoReader(os.path.join(self.path_to_input, file))
+            self.total_samples = self.total_samples + \
+                int(np.fix(info.data.frame_count/self.len_of_samples))
+    
+         
+    def create_generator(self):
+        '''
+        Method to create the iterator. 
+        '''
+
+        # check if training or validation
+        if self.train_flag:
+            shuffle(self.file_names)
+        # iterate over the files  
+        for file in self.file_names:
+            # read the audio files
+            noisy, fs_1 = sf.read(os.path.join(self.path_to_input, file))
+            speech, fs_2 = sf.read(os.path.join(self.path_to_s1, file))
+            # check if the sampling rates are matching the specifications
+            if fs_1 != self.fs or fs_2 != self.fs:
+                raise ValueError('Sampling rates do not match.')
+            if noisy.ndim != 1 or speech.ndim != 1:
+                raise ValueError('Too many audio channels. The DTLN audio_generator \
+                                 only supports single channel audio data.')
+            # count the number of samples in one file
+            num_samples = int(np.fix(noisy.shape[0]/self.len_of_samples))
+            # iterate over the number of samples
+            for idx in range(num_samples):
+                # cut the audio files in chunks
+                in_dat = noisy[int(idx*self.len_of_samples):int((idx+1)*
+                                                        self.len_of_samples)]
+                tar_dat = speech[int(idx*self.len_of_samples):int((idx+1)*
+                                                        self.len_of_samples)]
+                # yield the chunks as float32 data
+                yield in_dat.astype('float32'), tar_dat.astype('float32')
+              
+
+    def create_tf_data_obj(self):
+        '''
+        Method to to create the tf.data.Dataset. 
+        '''
+
+        # creating the tf.data.Dataset from the iterator
+        self.tf_data_set = tf.data.Dataset.from_generator(
+                        self.create_generator,
+                        (tf.float32, tf.float32),
+                        output_shapes=(tf.TensorShape([self.len_of_samples]), \
+                                       tf.TensorShape([self.len_of_samples])),
+                        args=None
+                        )
+
+        
+                
+
+
+class DTLN_model():
+    '''
+    Class to create and train the DTLN model
+    '''
+    
+    def __init__(self):
+        '''
+        Constructor
+        '''
+
+        # defining default cost function
+        self.cost_function = self.snr_cost
+        # empty property for the model
+        self.model = []
+        # defining default parameters
+        self.fs = 16000
+        self.batchsize = 32
+        self.len_samples = 15
+        self.activation = 'sigmoid'
+        self.numUnits = 128
+        self.numLayer = 2
+        self.blockLen = 512
+        self.block_shift = 128
+        self.dropout = 0.25
+        self.lr = 1e-3
+        self.max_epochs = 200
+        self.encoder_size = 256
+        self.eps = 1e-7
+        # reset all seeds to 42 to reduce invariance between training runs
+        os.environ['PYTHONHASHSEED']=str(42)
+        seed(42)
+        np.random.seed(42)
+        tf.random.set_seed(42)
+        # some line to correctly find some libraries in TF 2.x
+        physical_devices = tf.config.experimental.list_physical_devices('GPU')
+        if len(physical_devices) > 0:
+            for device in physical_devices:
+                tf.config.experimental.set_memory_growth(device, enable=True)
+        
+
+    @staticmethod
+    def snr_cost(s_estimate, s_true):
+        '''
+        Static Method defining the cost function. 
+        The negative signal to noise ratio is calculated here. The loss is 
+        always calculated over the last dimension. 
+        '''
+
+        # calculating the SNR
+        snr = tf.reduce_mean(tf.math.square(s_true), axis=-1, keepdims=True) / \
+            (tf.reduce_mean(tf.math.square(s_true-s_estimate), axis=-1, keepdims=True)+1e-7)
+        # using some more lines, because TF has no log10
+        num = tf.math.log(snr) 
+        denom = tf.math.log(tf.constant(10, dtype=num.dtype))
+        loss = -10*(num / (denom))
+        # returning the loss
+        return loss
+        
+
+    def lossWrapper(self):
+        '''
+        A wrapper function which returns the loss function. This is done to
+        to enable additional arguments to the loss function if necessary.
+        '''
+        def lossFunction(y_true,y_pred):
+            # calculating loss and squeezing single dimensions away
+            loss = tf.squeeze(self.cost_function(y_pred,y_true))
+            # calculate mean over batches
+            loss = tf.reduce_mean(loss)
+            # return the loss
+            return loss
+        # returning the loss function as handle
+        return lossFunction
+    
+    
+
+    '''
+    In the following some helper layers are defined.
+    '''  
+    
+    def stftLayer(self, x):
+        '''
+        Method for an STFT helper layer used with a Lambda layer. The layer
+        calculates the STFT on the last dimension and returns the magnitude and
+        phase of the STFT.
+        '''
+        
+        # creating frames from the continuous waveform
+        frames = tf.signal.frame(x, self.blockLen, self.block_shift)
+        # calculating the fft over the time frames. rfft returns NFFT/2+1 bins.
+        stft_dat = tf.signal.rfft(frames)
+        # calculating magnitude and phase from the complex signal
+        mag = tf.abs(stft_dat)
+        phase = tf.math.angle(stft_dat)
+        # returning magnitude and phase as list
+        return [mag, phase]
+    
+    def fftLayer(self, x):
+        '''
+        Method for an fft helper layer used with a Lambda layer. The layer
+        calculates the rFFT on the last dimension and returns the magnitude and
+        phase of the STFT.
+        '''
+        
+        # expanding dimensions
+        frame = tf.expand_dims(x, axis=1)
+        # calculating the fft over the time frames. rfft returns NFFT/2+1 bins.
+        stft_dat = tf.signal.rfft(frame)
+        # calculating magnitude and phase from the complex signal
+        mag = tf.abs(stft_dat)
+        phase = tf.math.angle(stft_dat)
+        # returning magnitude and phase as list
+        return [mag, phase]
+
+ 
+        
+    def ifftLayer(self, x):
+        '''
+        Method for an inverse FFT layer used with an Lambda layer. This layer
+        calculates time domain frames from magnitude and phase information. 
+        As input x a list with [mag,phase] is required.
+        '''
+        
+        # calculating the complex representation
+        s1_stft = (tf.cast(x[0], tf.complex64) * 
+                    tf.exp( (1j * tf.cast(x[1], tf.complex64))))
+        # returning the time domain frames
+        return tf.signal.irfft(s1_stft)  
+    
+    
+    def overlapAddLayer(self, x):
+        '''
+        Method for an overlap and add helper layer used with a Lambda layer.
+        This layer reconstructs the waveform from a framed signal.
+        '''
+
+        # calculating and returning the reconstructed waveform
+        return tf.signal.overlap_and_add(x, self.block_shift)
+    
+        
+
+    def seperation_kernel(self, num_layer, mask_size, x, stateful=False):
+        '''
+        Method to create a separation kernel. 
+        !! Important !!: Do not use this layer with a Lambda layer. If used with
+        a Lambda layer the gradients are updated correctly.
+
+        Inputs:
+            num_layer       Number of LSTM layers
+            mask_size       Output size of the mask and size of the Dense layer
+        '''
+
+        # creating num_layer number of LSTM layers
+        for idx in range(num_layer):
+            x = LSTM(self.numUnits, return_sequences=True, stateful=stateful)(x)
+            # using dropout between the LSTM layer for regularization 
+            if idx<(num_layer-1):
+                x = Dropout(self.dropout)(x)
+        # creating the mask with a Dense and an Activation layer
+        mask = Dense(mask_size)(x)
+        mask = Activation(self.activation)(mask)
+        # returning the mask
+        return mask
+    
+    def seperation_kernel_with_states(self, num_layer, mask_size, x, 
+                                      in_states):
+        '''
+        Method to create a separation kernel, which returns the LSTM states. 
+        !! Important !!: Do not use this layer with a Lambda layer. If used with
+        a Lambda layer the gradients are updated correctly.
+
+        Inputs:
+            num_layer       Number of LSTM layers
+            mask_size       Output size of the mask and size of the Dense layer
+        '''
+        
+        states_h = []
+        states_c = []
+        # creating num_layer number of LSTM layers
+        for idx in range(num_layer):
+            in_state = [in_states[:,idx,:, 0], in_states[:,idx,:, 1]]
+            x, h_state, c_state = LSTM(self.numUnits, return_sequences=True, 
+                     unroll=True, return_state=True)(x, initial_state=in_state)
+            # using dropout between the LSTM layer for regularization 
+            if idx<(num_layer-1):
+                x = Dropout(self.dropout)(x)
+            states_h.append(h_state)
+            states_c.append(c_state)
+        # creating the mask with a Dense and an Activation layer
+        mask = Dense(mask_size)(x)
+        mask = Activation(self.activation)(mask)
+        out_states_h = tf.reshape(tf.stack(states_h, axis=0), 
+                                  [1,num_layer,self.numUnits])
+        out_states_c = tf.reshape(tf.stack(states_c, axis=0), 
+                                  [1,num_layer,self.numUnits])
+        out_states = tf.stack([out_states_h, out_states_c], axis=-1)
+        # returning the mask and states
+        return mask, out_states
+
+    def build_DTLN_model(self, norm_stft=False):
+        '''
+        Method to build and compile the DTLN model. The model takes time domain 
+        batches of size (batchsize, len_in_samples) and returns enhanced clips 
+        in the same dimensions. As optimizer for the Training process the Adam
+        optimizer with a gradient norm clipping of 3 is used. 
+        The model contains two separation cores. The first has an STFT signal 
+        transformation and the second a learned transformation based on 1D-Conv 
+        layer. 
+        '''
+        
+        # input layer for time signal
+        time_dat = Input(batch_shape=(None, None))
+        # calculate STFT
+        mag,angle = Lambda(self.stftLayer)(time_dat)
+        # normalizing log magnitude stfts to get more robust against level variations
+        if norm_stft:
+            mag_norm = InstantLayerNormalization()(tf.math.log(mag + 1e-7))
+        else:
+            # behaviour like in the paper
+            mag_norm = mag
+        # predicting mask with separation kernel  
+        mask_1 = self.seperation_kernel(self.numLayer, (self.blockLen//2+1), mag_norm)
+        # multiply mask with magnitude
+        estimated_mag = Multiply()([mag, mask_1])
+        # transform frames back to time domain
+        estimated_frames_1 = Lambda(self.ifftLayer)([estimated_mag,angle])
+        # encode time domain frames to feature domain
+        encoded_frames = Conv1D(self.encoder_size,1,strides=1,use_bias=False)(estimated_frames_1)
+        # normalize the input to the separation kernel
+        encoded_frames_norm = InstantLayerNormalization()(encoded_frames)
+        # predict mask based on the normalized feature frames
+        mask_2 = self.seperation_kernel(self.numLayer, self.encoder_size, encoded_frames_norm)
+        # multiply encoded frames with the mask
+        estimated = Multiply()([encoded_frames, mask_2]) 
+        # decode the frames back to time domain
+        decoded_frames = Conv1D(self.blockLen, 1, padding='causal',use_bias=False)(estimated)
+        # create waveform with overlap and add procedure
+        estimated_sig = Lambda(self.overlapAddLayer)(decoded_frames)
+
+        
+        # create the model
+        self.model = Model(inputs=time_dat, outputs=estimated_sig)
+        # show the model summary
+        print(self.model.summary())
+        
+    def build_DTLN_model_stateful(self, norm_stft=False):
+        '''
+        Method to build stateful DTLN model for real time processing. The model 
+        takes one time domain frame of size (1, blockLen) and one enhanced frame. 
+         
+        '''
+        
+        # input layer for time signal
+        time_dat = Input(batch_shape=(1, self.blockLen))
+        # calculate STFT
+        mag,angle = Lambda(self.fftLayer)(time_dat)
+        # normalizing log magnitude stfts to get more robust against level variations
+        if norm_stft:
+            mag_norm = InstantLayerNormalization()(tf.math.log(mag + 1e-7))
+        else:
+            # behaviour like in the paper
+            mag_norm = mag
+        # predicting mask with separation kernel  
+        mask_1 = self.seperation_kernel(self.numLayer, (self.blockLen//2+1), mag_norm, stateful=True)
+        # multiply mask with magnitude
+        estimated_mag = Multiply()([mag, mask_1])
+        # transform frames back to time domain
+        estimated_frames_1 = Lambda(self.ifftLayer)([estimated_mag,angle])
+        # encode time domain frames to feature domain
+        encoded_frames = Conv1D(self.encoder_size,1,strides=1,use_bias=False)(estimated_frames_1)
+        # normalize the input to the separation kernel
+        encoded_frames_norm = InstantLayerNormalization()(encoded_frames)
+        # predict mask based on the normalized feature frames
+        mask_2 = self.seperation_kernel(self.numLayer, self.encoder_size, encoded_frames_norm, stateful=True)
+        # multiply encoded frames with the mask
+        estimated = Multiply()([encoded_frames, mask_2]) 
+        # decode the frames back to time domain
+        decoded_frame = Conv1D(self.blockLen, 1, padding='causal',use_bias=False)(estimated)
+        # create the model
+        self.model = Model(inputs=time_dat, outputs=decoded_frame)
+        # show the model summary
+        print(self.model.summary())
+        
+    def compile_model(self):
+        '''
+        Method to compile the model for training
+
+        '''
+        
+        # use the Adam optimizer with a clipnorm of 3
+        optimizerAdam = keras.optimizers.Adam(lr=self.lr, clipnorm=3.0)
+        # compile model with loss function
+        self.model.compile(loss=self.lossWrapper(), optimizer=optimizerAdam)
+        
+    def create_saved_model(self, weights_file, target_name):
+        '''
+        Method to create a saved model folder from a weights file
+
+        '''
+        # check for type
+        if weights_file.find('_norm_') != -1:
+            norm_stft = True
+        else:
+            norm_stft = False
+        # build model    
+        self.build_DTLN_model_stateful(norm_stft=norm_stft)
+        # load weights
+        self.model.load_weights(weights_file)
+        # save model
+        tf.saved_model.save(self.model, target_name)
+        
+    def create_tf_lite_model(self, weights_file, target_name, use_dynamic_range_quant=False):
+        '''
+        Method to create a tf lite model folder from a weights file. 
+        The conversion creates two models, one for each separation core. 
+        Tf lite does not support complex numbers yet. Some processing must be 
+        done outside the model.
+        For further information and how real time processing can be 
+        implemented see "real_time_processing_tf_lite.py".
+        
+        The conversion only works with TF 2.3.
+
+        '''
+        # check for type
+        if weights_file.find('_norm_') != -1:
+            norm_stft = True
+            num_elements_first_core = 2 + self.numLayer * 3 + 2
+        else:
+            norm_stft = False
+            num_elements_first_core = self.numLayer * 3 + 2
+        # build model    
+        self.build_DTLN_model_stateful(norm_stft=norm_stft)
+        # load weights
+        self.model.load_weights(weights_file)
+        
+        #### Model 1 ##########################
+        mag = Input(batch_shape=(1, 1, (self.blockLen//2+1)))
+        states_in_1 = Input(batch_shape=(1, self.numLayer, self.numUnits, 2))
+        # normalizing log magnitude stfts to get more robust against level variations
+        if norm_stft:
+            mag_norm = InstantLayerNormalization()(tf.math.log(mag + 1e-7))
+        else:
+            # behaviour like in the paper
+            mag_norm = mag
+        # predicting mask with separation kernel  
+        mask_1, states_out_1 = self.seperation_kernel_with_states(self.numLayer, 
+                                                    (self.blockLen//2+1), 
+                                                    mag_norm, states_in_1)
+        
+        model_1 = Model(inputs=[mag, states_in_1], outputs=[mask_1, states_out_1])
+        
+        #### Model 2 ###########################
+        
+        estimated_frame_1 = Input(batch_shape=(1, 1, (self.blockLen)))
+        states_in_2 = Input(batch_shape=(1, self.numLayer, self.numUnits, 2))
+        
+        # encode time domain frames to feature domain
+        encoded_frames = Conv1D(self.encoder_size,1,strides=1,
+                                use_bias=False)(estimated_frame_1)
+        # normalize the input to the separation kernel
+        encoded_frames_norm = InstantLayerNormalization()(encoded_frames)
+        # predict mask based on the normalized feature frames
+        mask_2, states_out_2 = self.seperation_kernel_with_states(self.numLayer, 
+                                                    self.encoder_size, 
+                                                    encoded_frames_norm, 
+                                                    states_in_2)
+        # multiply encoded frames with the mask
+        estimated = Multiply()([encoded_frames, mask_2]) 
+        # decode the frames back to time domain
+        decoded_frame = Conv1D(self.blockLen, 1, padding='causal',
+                               use_bias=False)(estimated)
+        
+        model_2 = Model(inputs=[estimated_frame_1, states_in_2], 
+                        outputs=[decoded_frame, states_out_2])
+        
+        # set weights to submodels
+        weights = self.model.get_weights()
+        model_1.set_weights(weights[:num_elements_first_core])
+        model_2.set_weights(weights[num_elements_first_core:])
+        # convert first model
+        converter = tf.lite.TFLiteConverter.from_keras_model(model_1)
+        if use_dynamic_range_quant:
+            converter.optimizations = [tf.lite.Optimize.DEFAULT]
+        tflite_model = converter.convert()
+        with tf.io.gfile.GFile(target_name + '_1.tflite', 'wb') as f:
+              f.write(tflite_model)
+        # convert second model    
+        converter = tf.lite.TFLiteConverter.from_keras_model(model_2)
+        if use_dynamic_range_quant:
+            converter.optimizations = [tf.lite.Optimize.DEFAULT]
+        tflite_model = converter.convert()
+        with tf.io.gfile.GFile(target_name + '_2.tflite', 'wb') as f:
+              f.write(tflite_model)
+              
+        print('TF lite conversion complete!')
+        
+    
+    def train_model(self, runName, path_to_train_mix, path_to_train_speech, \
+                    path_to_val_mix, path_to_val_speech):
+        '''
+        Method to train the DTLN model. 
+        '''
+        
+        # create save path if not existent
+        savePath = './models_'+ runName+'/' 
+        if not os.path.exists(savePath):
+            os.makedirs(savePath)
+        # create log file writer
+        csv_logger = CSVLogger(savePath+ 'training_' +runName+ '.log')
+        # create callback for the adaptive learning rate
+        reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5,
+                              patience=3, min_lr=10**(-10), cooldown=1)
+        # create callback for early stopping
+        early_stopping = EarlyStopping(monitor='val_loss', min_delta=0, 
+            patience=10, verbose=0, mode='auto', baseline=None)
+        # create model check pointer to save the best model
+        checkpointer = ModelCheckpoint(savePath+runName+'.h5',
+                                       monitor='val_loss',
+                                       verbose=1,
+                                       save_best_only=True,
+                                       save_weights_only=True,
+                                       mode='auto',
+                                       save_freq='epoch'
+                                       )
+
+        # calculate length of audio chunks in samples
+        len_in_samples = int(np.fix(self.fs * self.len_samples / 
+                                    self.block_shift)*self.block_shift)
+        # create data generator for training data
+        generator_input = audio_generator(path_to_train_mix, 
+                                          path_to_train_speech, 
+                                          len_in_samples, 
+                                          self.fs, train_flag=True)
+        dataset = generator_input.tf_data_set
+        dataset = dataset.batch(self.batchsize, drop_remainder=True).repeat()
+        # calculate number of training steps in one epoch
+        steps_train = generator_input.total_samples//self.batchsize
+        # create data generator for validation data
+        generator_val = audio_generator(path_to_val_mix,
+                                        path_to_val_speech, 
+                                        len_in_samples, self.fs)
+        dataset_val = generator_val.tf_data_set
+        dataset_val = dataset_val.batch(self.batchsize, drop_remainder=True).repeat()
+        # calculate number of validation steps
+        steps_val = generator_val.total_samples//self.batchsize
+        # start the training of the model
+        self.model.fit(
+            x=dataset, 
+            batch_size=None,
+            steps_per_epoch=steps_train, 
+            epochs=self.max_epochs,
+            verbose=1,
+            validation_data=dataset_val,
+            validation_steps=steps_val, 
+            callbacks=[checkpointer, reduce_lr, csv_logger, early_stopping],
+            max_queue_size=50,
+            workers=4,
+            use_multiprocessing=True)
+        # clear out garbage
+        tf.keras.backend.clear_session()
+
+    
+
+class InstantLayerNormalization(Layer):
+    '''
+    Class implementing instant layer normalization. It can also be called 
+    channel-wise layer normalization and was proposed by 
+    Luo & Mesgarani (https://arxiv.org/abs/1809.07454v2) 
+    '''
+
+    def __init__(self, **kwargs):
+        '''
+            Constructor
+        '''
+        super(InstantLayerNormalization, self).__init__(**kwargs)
+        self.epsilon = 1e-7 
+        self.gamma = None
+        self.beta = None
+
+    def build(self, input_shape):
+        '''
+        Method to build the weights.
+        '''
+        shape = input_shape[-1:]
+        # initialize gamma
+        self.gamma = self.add_weight(shape=shape,
+                             initializer='ones',
+                             trainable=True,
+                             name='gamma')
+        # initialize beta
+        self.beta = self.add_weight(shape=shape,
+                             initializer='zeros',
+                             trainable=True,
+                             name='beta')
+ 
+
+    def call(self, inputs):
+        '''
+        Method to call the Layer. All processing is done here.
+        '''
+
+        # calculate mean of each frame
+        mean = tf.math.reduce_mean(inputs, axis=[-1], keepdims=True)
+        # calculate variance of each frame
+        variance = tf.math.reduce_mean(tf.math.square(inputs - mean), 
+                                       axis=[-1], keepdims=True)
+        # calculate standard deviation
+        std = tf.math.sqrt(variance + self.epsilon)
+        # normalize each frame independently 
+        outputs = (inputs - mean) / std
+        # scale with gamma
+        outputs = outputs * self.gamma
+        # add the bias beta
+        outputs = outputs + self.beta
+        # return output
+        return outputs
+    
+
+    

models/DTLN (alekya)/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Nils L. Westhausen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

models/DTLN (alekya)/README.md

@@ -0,0 +1,302 @@
+# Dual-signal Transformation LSTM Network
+
++ Tensorflow 2.x implementation of the stacked dual-signal transformation LSTM network (DTLN) for real-time noise suppression.
++ This repository provides the code for training, infering and serving the DTLN model in python. It also provides pretrained models in SavedModel, TF-lite and ONNX format, which can be used as baseline for your own projects. The model is able to run with real time audio on a RaspberryPi.
++ If you are doing cool things with this repo, tell me about it. I am always curious about what you are doing with this code or this models.
+
+---
+
+The DTLN model was handed in to the deep noise suppression challenge ([DNS-Challenge](https://github.com/microsoft/DNS-Challenge)) and the paper was presented at Interspeech 2020. 
+
+
+This approach combines a short-time Fourier transform (STFT) and a learned analysis and synthesis basis in a stacked-network approach with less than one million parameters. The model was trained on 500h of noisy speech provided by the challenge organizers. The network is capable of real-time processing (one frame in, one frame out) and reaches competitive results.
+Combining these two types of signal transformations enables the DTLN to robustly extract information from magnitude spectra and incorporate phase information from the learned feature basis. The method shows state-of-the-art performance and outperforms the DNS-Challenge baseline by 0.24 points absolute in terms of the mean opinion score (MOS).
+
+For more information see the [paper](https://www.isca-speech.org/archive/interspeech_2020/westhausen20_interspeech.html). The results of the DNS-Challenge are published [here](https://www.microsoft.com/en-us/research/academic-program/deep-noise-suppression-challenge-interspeech-2020/#!results). We reached a competitive 8th place out of 17 teams in the real time track.
+
+---
+
+For baseline usage and to reproduce the processing used for the paper run:
+```bash
+$ python run_evaluation.py -i in/folder/with/wav -o target/folder/processed/files -m ./pretrained_model/model.h5
+```
+
+---
+
+The pretrained DTLN-aec (the DTLN applied to acoustic echo cancellation) can be found in the [DTLN-aec repository](https://github.com/breizhn/DTLN-aec).
+
+---
+
+Author: Nils L. Westhausen ([Communication Acoustics](https://uol.de/en/kommunikationsakustik) , Carl von Ossietzky University, Oldenburg, Germany)
+
+This code is licensed under the terms of the MIT license.
+
+
+---
+### Citing:
+
+If you are using the DTLN model, please cite:
+
+```BibTex
+@inproceedings{Westhausen2020,
+  author={Nils L. Westhausen and Bernd T. Meyer},
+  title={{Dual-Signal Transformation LSTM Network for Real-Time Noise Suppression}},
+  year=2020,
+  booktitle={Proc. Interspeech 2020},
+  pages={2477--2481},
+  doi={10.21437/Interspeech.2020-2631},
+  url={http://dx.doi.org/10.21437/Interspeech.2020-2631}
+}
+```
+
+
+---
+### Contents of the README:
+
+* [Results](#results)
+* [Execution Times](#execution-times)
+* [Audio Samples](#audio-samples)
+* [Contents of the repository](#contents-of-the-repository)
+* [Python dependencies](#python-dependencies)
+* [Training data preparation](#training-data-preparation)
+* [Run a training of the DTLN model](#run-a-training-of-the-dtln-model)
+* [Measuring the execution time of the DTLN model with the SavedModel format](#measuring-the-execution-time-of-the-dtln-model-with-the-savedmodel-format)
+* [Real time processing with the SavedModel format](#real-time-processing-with-the-savedmodel-format)
+* [Real time processing with tf-lite](#real-time-processing-with-tf-lite)
+* [Real time audio with sounddevice and tf-lite](#real-time-audio-with-sounddevice-and-tf-lite)
+* [Model conversion and real time processing with ONNX](#model-conversion-and-real-time-processing-with-onnx)
+
+
+
+---
+### Results:
+
+Results on the DNS-Challenge non reverberant test set:
+Model | PESQ [mos] | STOI [%] | SI-SDR [dB] | TF version
+--- | --- | --- | --- | ---
+unprocessed | 2.45 | 91.52 | 9.07 |
+NsNet (Baseline) | 2.70 | 90.56 | 12.57 |
+ |  |  |  | 
+DTLN (500h) | 3.04 | 94.76 | 16.34 | 2.1
+DTLN (500h)| 2.98 | 94.75 | 16.20 | TF-light
+DTLN (500h) | 2.95 | 94.47 | 15.71 | TF-light quantized
+ |  |  |  | 
+DTLN norm (500h) | 3.04 | 94.47 | 16.10 | 2.2
+ |  |  |  | 
+DTLN norm (40h) | 3.05 | 94.57 | 16.88 | 2.2
+DTLN norm (40h) | 2.98 | 94.56 | 16.58 | TF-light
+DTLN norm (40h) | 2.98 | 94.51 | 16.22 | TF-light quantized
+
+* The conversion to TF-light slightly reduces the performance. 
+* The dynamic range quantization of TF-light also reduces the performance a bit and introduces some quantization noise. But the audio-quality is still on a high level and the model is real-time capable on the Raspberry Pi 3 B+.
+* The normalization of the log magnitude of the STFT does not decrease the model performance and makes it more robust against level variations.
+* With data augmentation during training it is possible to train the DTLN model on just 40h of noise and speech data. If you have any question regarding this, just contact me.
+
+[To contents](#contents-of-the-readme)
+
+---
+
+### Execution Times:
+
+Execution times for SavedModel are measured with TF 2.2 and for TF-lite with the TF-lite runtime:
+System | Processor | #Cores | SavedModel | TF-lite | TF-lite quantized
+--- | --- | --- | --- | --- | ---
+Ubuntu 18.04         | Intel I5 6600k @ 3.5 GHz | 4 | 0.65 ms | 0.36 ms | 0.27 ms
+Macbook Air mid 2012 | Intel I7 3667U @ 2.0 GHz | 2 | 1.4 ms | 0.6 ms | 0.4 ms
+Raspberry Pi 3 B+    | ARM Cortex A53 @ 1.4 GHz | 4 | 15.54 ms | 9.6 ms | 2.2 ms
+
+For real-time capability the execution time must be below 8 ms.
+
+[To contents](#contents-of-the-readme)
+
+---
+
+### Audio Samples:
+
+Here some audio samples created with the tf-lite model. Sadly audio can not be integrated directly into markdown.
+
+Noisy | Enhanced | Noise type
+--- | --- | --- 
+[Sample 1](https://cloudsync.uol.de/s/GFHzmWWJAwgQPLf) | [Sample 1](https://cloudsync.uol.de/s/p3M48y7cjkJ2ZZg) | Air conditioning
+[Sample 2](https://cloudsync.uol.de/s/4Y2PoSpJf7nXx9T) | [Sample 2](https://cloudsync.uol.de/s/QeK4aH5KCELPnko) | Music
+[Sample 3](https://cloudsync.uol.de/s/Awc6oBtnTpb5pY7) | [Sample 3](https://cloudsync.uol.de/s/yNsmDgxH3MPWMTi) | Bus 
+
+
+[To contents](#contents-of-the-readme)
+
+---
+### Contents of the repository:
+
+*  **DTLN_model.py** \
+  This file is containing the model, data generator and the training routine.
+*  **run_training.py** \
+  Script to run the training. Before you can start the training with `$ python run_training.py`you have to set the paths to you training and validation data inside the script. The training script uses a default setup.
+* **run_evaluation.py** \
+  Script to process a folder with optional subfolders containing .wav files with a trained DTLN model. With the pretrained model delivered with this repository a folder can be processed as following: \
+  `$ python run_evaluation.py -i /path/to/input -o /path/for/processed -m ./pretrained_model/model.h5` \
+  The evaluation script will create the new folder with the same structure as the input folder and the files will have the same name as the input files.
+* **measure_execution_time.py** \
+  Script for measuring the execution time with the saved DTLN model in `./pretrained_model/dtln_saved_model/`. For further information see this [section](#measuring-the-execution-time-of-the-dtln-model-with-the-savedmodel-format).
+* **real_time_processing.py** \
+  Script, which explains how real time processing with the SavedModel works. For more information see this [section](#real-time-processing-with-the-savedmodel-format).
++  **./pretrained_model/** \
+   * `model.h5`: Model weights as used in the DNS-Challenge DTLN model.
+   * `DTLN_norm_500h.h5`: Model weights trained on 500h with normalization of stft log magnitudes.
+   * `DTLN_norm_40h.h5`: Model weights trained on 40h with normalization of stft log magnitudes.
+   * `./dtln_saved_model`: same as `model.h5` but as a stateful model in SavedModel format.
+   * `./DTLN_norm_500h_saved_model`: same as `DTLN_norm_500h.h5` but as a stateful model in SavedModel format.
+   * `./DTLN_norm_40h_saved_model`: same as `DTLN_norm_40h.h5` but as a stateful model in SavedModel format.
+   * `model_1.tflite` together with `model_2.tflite`: same as `model.h5` but as TF-lite model with external state handling.
+   * `model_quant_1.tflite` together with `model_quant_2.tflite`: same as `model.h5` but as TF-lite model with external state handling and dynamic range quantization.
+   * `model_1.onnx` together with `model_2.onnx`: same as `model.h5` but as ONNX model with external state handling.
+   
+[To contents](#contents-of-the-readme)
+   
+---
+### Python dependencies:
+
+The following packages will be required for this repository:
+* TensorFlow (2.x)
+* librosa
+* wavinfo 
+
+
+All additional packages (numpy, soundfile, etc.) should be installed on the fly when using conda or pip. I recommend using conda environments or [pyenv](https://github.com/pyenv/pyenv) [virtualenv](https://github.com/pyenv/pyenv-virtualenv) for the python environment. For training a GPU with at least 5 GB of memory is required. I recommend at least Tensorflow 2.1 with Nvidia driver 418 and Cuda 10.1. If you use conda Cuda will be installed on the fly and you just need the driver. For evaluation-only the CPU version of Tensorflow is enough. Everything was tested on Ubuntu 18.04.
+
+Conda environments for training (with cuda) and for evaluation (CPU only) can be created as following:
+
+For the training environment:
+```shell
+$ conda env create -f train_env.yml
+```
+For the evaluation environment:
+```
+$ conda env create -f eval_env.yml
+```
+For the tf-lite environment:
+```
+$ conda env create -f tflite_env.yml
+```
+The tf-lite runtime must be downloaded from [here](https://www.tensorflow.org/lite/guide/python).
+
+[To contents](#contents-of-the-readme)
+
+---
+### Training data preparation:
+
+1. Clone the forked DNS-Challenge [repository](https://github.com/breizhn/DNS-Challenge). Before cloning the repository make sure `git-lfs` is installed. Also make sure your disk has enough space. I recommend downloading the data to an SSD for faster dataset creation.
+
+2. Run `noisyspeech_synthesizer_multiprocessing.py` to create the dataset. `noisyspeech_synthesizer.cfg`was changed according to my training setup used for the DNS-Challenge. 
+
+3. Run `split_dns_corpus.py`to divide the dataset in training and validation data. The classic 80:20 split is applied. This file was added to the forked repository by me.
+
+[To contents](#contents-of-the-readme)
+
+---
+### Run a training of the DTLN model:
+
+1. Make sure all dependencies are installed in your python environment.
+
+2. Change the paths to your training and validation dataset in `run_training.py`.
+
+3. Run `$ python run_training.py`. 
+
+One epoch takes around 21 minutes on a Nvidia RTX 2080 Ti when loading the training data from an SSD. 
+
+[To contents](#contents-of-the-readme)
+
+---
+### Measuring the execution time of the DTLN model with the SavedModel format:
+
+In total there are three ways to measure the execution time for one block of the model: Running a sequence in Keras and dividing by the number of blocks in the sequence, building a stateful model in Keras and running block by block, and saving the stateful model in Tensorflow's SavedModel format and calling that one block by block. In the following I will explain how running the model in the SavedModel format, because it is the most portable version and can also be called from Tensorflow Serving.
+
+A Keras model can be saved to the saved model format:
+```python
+import tensorflow as tf
+'''
+Building some model here
+'''
+tf.saved_model.save(your_keras_model, 'name_save_path')
+```
+Important here for real time block by block processing is, to make the LSTM layer stateful, so they can remember the states from the previous block.
+
+The model can be imported with 
+```python
+model = tf.saved_model.load('name_save_path')
+```
+
+For inference we now first call this for mapping signature names to functions
+```python
+infer = model.signatures['serving_default']
+```
+
+and now for inferring the block `x` call
+```python
+y = infer(tf.constant(x))['conv1d_1']
+```
+This command gives you the result on the node `'conv1d_1'`which is our output node for real time processing. For more information on using the SavedModel format and obtaining the output node see this [Guide](https://www.tensorflow.org/guide/saved_model).
+
+For making everything easier this repository provides a stateful DTLN SavedModel. 
+For measuring the execution time call:
+```
+$ python measure_execution_time.py
+```
+
+[To contents](#contents-of-the-readme)
+
+---
+
+### Real time processing with the SavedModel format:
+
+For explanation look at `real_time_processing.py`. 
+
+Here some consideration for integrating this model in your project:
+* The sampling rate of this model is fixed at 16 kHz. It will not work smoothly with other sampling rates.
+* The block length of 32 ms and the block shift of 8 ms are also fixed. For changing these values, the model must be retrained.
+* The delay created by the model is the block length, so the input-output delay is 32 ms.
+* For real time capability on your system, the execution time must be below the length of the block shift, so below 8 ms. 
+* If can not give you support on the hardware side, regarding soundcards, drivers and so on. Be aware, a lot of artifacts can come from this side.
+
+[To contents](#contents-of-the-readme)
+
+---
+### Real time processing with tf-lite:
+
+With TF 2.3 it is finally possible to convert LSTMs to tf-lite. It is still not perfect because the states must be handled seperatly for a stateful model and tf-light does not support complex numbers. That means that the model is splitted in two submodels when converting it to tf-lite and the calculation of the FFT and iFFT is performed outside the model. I provided an example script for explaining, how real time processing with the tf light model works (```real_time_processing_tf_lite.py```). In this script the tf-lite runtime is used. The runtime can be downloaded [here](https://www.tensorflow.org/lite/guide/python). Quantization works now.
+
+Using the tf-lite DTLN model and the tf-lite runtime the execution time on an old Macbook Air mid 2012 can be decreased to **0.6 ms**.
+
+[To contents](#contents-of-the-readme)
+
+---
+### Real time audio with sounddevice and tf-lite:
+
+The file ```real_time_dtln_audio.py```is an example how real time audio with the tf-lite model and the [sounddevice](https://github.com/spatialaudio/python-sounddevice) toolbox can be implemented. The script is based on the ```wire.py``` example. It works fine on an old Macbook Air mid 2012 and so it will probably run on most newer devices. In the quantized version it was sucessfully tested on an Raspberry Pi 3B +.
+
+First check for your audio devices:
+```
+$ python real_time_dtln_audio.py --list-devices
+```
+Choose the index of an input and an output device and call:
+```
+$ python real_time_dtln_audio.py -i in_device_idx -o out_device_idx
+```
+If the script is showing too much ```input underflow``` restart the sript. If that does not help, increase the latency with the ```--latency``` option. The default value is 0.2 .
+
+[To contents](#contents-of-the-readme)
+
+---
+### Model conversion and real time processing with ONNX:
+
+Finally I got the ONNX model working. 
+For converting the model TF 2.1 and keras2onnx is required. keras2onnx can be downloaded [here](https://github.com/onnx/keras-onnx) and must be installed from source as described in the README. When all dependencies are installed, call:
+```
+$ python convert_weights_to_onnx.py -m /name/of/the/model.h5 -t onnx_model_name
+```
+to convert the model to the ONNX format. The model is split in two parts as for the TF-lite model. The conversion does not work on MacOS.
+The real time processing works similar to the TF-lite model and can be looked up in following file: ```real_time_processing_onnx.py ```
+The ONNX runtime required for this script can be installed with:
+```
+$ pip install onnxruntime
+```
+The execution time on the Macbook Air mid 2012 is around 1.13 ms for one block.

models/DTLN (alekya)/convert_weights_to_onnx.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+"""
+Script to covert a .h5 weights file of the DTLN model to ONNX.
+At the moment the conversion only works with TF 2.1 and not on Mac.
+
+Example call:
+    $python convert_weights_to_ONNX.py -m /name/of/the/model.h5 \
+                                              -t name_target 
+                              
+
+Author: Nils L. Westhausen (nils.westhausen@uol.de)
+Version: 03.07.2020
+
+This code is licensed under the terms of the MIT-license.
+"""
+
+from DTLN_model import DTLN_model, InstantLayerNormalization
+import argparse
+from tensorflow.keras.models import Model
+from tensorflow.keras.layers import Input, Multiply, Conv1D
+import tensorflow as tf
+import keras2onnx
+
+
+if __name__ == '__main__':
+    # arguement parser for running directly from the command line
+    parser = argparse.ArgumentParser(description='data evaluation')
+    parser.add_argument('--weights_file', '-m',
+                        help='path to .h5 weights file')
+    parser.add_argument('--target_folder', '-t',
+                        help='target folder for saved model')
+    
+    args = parser.parse_args()
+    weights_file = args.weights_file
+    dtln_class = DTLN_model()
+    # check for type
+    if weights_file.find('_norm_') != -1:
+        norm_stft = True
+        num_elements_first_core = 2 + dtln_class.numLayer * 3 + 2
+    else:
+        norm_stft = False
+        num_elements_first_core = dtln_class.numLayer * 3 + 2
+    # build model    
+    dtln_class.build_DTLN_model_stateful(norm_stft=norm_stft)
+    # load weights
+    dtln_class.model.load_weights(weights_file)
+    #### Model 1 ##########################
+    mag = Input(batch_shape=(1, 1, (dtln_class.blockLen//2+1)))
+    states_in_1 = Input(batch_shape=(1, dtln_class.numLayer, dtln_class.numUnits, 2))
+    # normalizing log magnitude stfts to get more robust against level variations
+    if norm_stft:
+        mag_norm = InstantLayerNormalization()(tf.math.log(mag + 1e-7))
+    else:
+        # behaviour like in the paper
+        mag_norm = mag
+    # predicting mask with separation kernel  
+    mask_1, states_out_1 = dtln_class.seperation_kernel_with_states(dtln_class.numLayer, 
+                                                (dtln_class.blockLen//2+1), 
+                                                mag_norm, states_in_1)
+    
+    model_1 = Model(inputs=[mag, states_in_1], outputs=[mask_1, states_out_1])
+    
+    #### Model 2 ###########################
+    
+    estimated_frame_1 = Input(batch_shape=(1, 1, (dtln_class.blockLen)))
+    states_in_2 = Input(batch_shape=(1, dtln_class.numLayer, dtln_class.numUnits, 2))
+    
+    # encode time domain frames to feature domain
+    encoded_frames = Conv1D(dtln_class.encoder_size,1,strides=1,
+                            use_bias=False)(estimated_frame_1)
+    # normalize the input to the separation kernel
+    encoded_frames_norm = InstantLayerNormalization()(encoded_frames)
+    # predict mask based on the normalized feature frames
+    mask_2, states_out_2 = dtln_class.seperation_kernel_with_states(dtln_class.numLayer, 
+                                                dtln_class.encoder_size, 
+                                                encoded_frames_norm, 
+                                                states_in_2)
+    # multiply encoded frames with the mask
+    estimated = Multiply()([encoded_frames, mask_2]) 
+    # decode the frames back to time domain
+    decoded_frame = Conv1D(dtln_class.blockLen, 1, padding='causal',
+                           use_bias=False)(estimated)
+    
+    model_2 = Model(inputs=[estimated_frame_1, states_in_2], 
+                    outputs=[decoded_frame, states_out_2])
+    
+    # set weights to submodels
+    weights = dtln_class.model.get_weights()
+    
+    model_1.set_weights(weights[:num_elements_first_core])
+    model_2.set_weights(weights[num_elements_first_core:])
+    # convert first model
+    onnx_model = keras2onnx.convert_keras(model_1)
+    temp_model_file = args.target_folder + '_1.onnx'
+    keras2onnx.save_model(onnx_model, temp_model_file)
+    # convert second model
+    onnx_model = keras2onnx.convert_keras(model_2)
+    temp_model_file = args.target_folder + '_2.onnx'
+    keras2onnx.save_model(onnx_model, temp_model_file)
+    
+          
+    print('ONNX conversion complete!')

models/DTLN (alekya)/convert_weights_to_saved_model.py

@@ -0,0 +1,30 @@
+"""
+Script to covert a .h weights file of the DTLN model to the saved model format.
+
+Example call:
+    $python convert_weights_to_saved_model.py -m /name/of/the/model.h5 \
+                                              -t name_target_folder 
+                              
+
+Author: Nils L. Westhausen (nils.westhausen@uol.de)
+Version: 24.06.2020
+
+This code is licensed under the terms of the MIT-license.
+"""
+
+from DTLN_model import DTLN_model
+import argparse
+
+
+if __name__ == '__main__':
+    # arguement parser for running directly from the command line
+    parser = argparse.ArgumentParser(description='data evaluation')
+    parser.add_argument('--weights_file', '-m',
+                        help='path to .h5 weights file')
+    parser.add_argument('--target_folder', '-t',
+                        help='target folder for saved model')
+    
+    args = parser.parse_args()
+    
+    converter = DTLN_model()
+    converter.create_saved_model(args.weights_file, args.target_folder)

models/DTLN (alekya)/convert_weights_to_tf_lite.py

@@ -0,0 +1,41 @@
+"""
+Script to covert a .h5 weights file of the DTLN model to tf lite.
+
+Example call:
+    $python convert_weights_to_tf_light.py -m /name/of/the/model.h5 \
+                                              -t name_target 
+                              
+
+Author: Nils L. Westhausen (nils.westhausen@uol.de)
+Version: 30.06.2020
+
+This code is licensed under the terms of the MIT-license.
+"""
+
+from DTLN_model import DTLN_model
+import argparse
+from pkg_resources import parse_version
+import tensorflow as tf
+
+
+if __name__ == '__main__':
+    # arguement parser for running directly from the command line
+    parser = argparse.ArgumentParser(description='data evaluation')
+    parser.add_argument('--weights_file', '-m',
+                        help='path to .h5 weights file')
+    parser.add_argument('--target_folder', '-t',
+                        help='target folder for saved model')
+    parser.add_argument('--quantization', '-q',
+                        help='use quantization (True/False)',
+                        default='False')
+    
+    args = parser.parse_args()
+    if parse_version(tf.__version__) < parse_version('2.3.0-rc0'):
+        raise ValueError('Tf version < 2.3. Conversion of LSTMs will not work'+ 
+                         +' with older tensorflow versions')
+    
+    
+    converter = DTLN_model()
+    converter.create_tf_lite_model(args.weights_file, 
+                                   args.target_folder, 
+                                   use_dynamic_range_quant=bool(args.quantization))

models/DTLN (alekya)/eval_env.yml

@@ -0,0 +1,117 @@
+name: eval_env
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=main
+  - _tflow_select=2.3.0=mkl
+  - absl-py=0.9.0=py37_0
+  - astunparse=1.6.3=py_0
+  - audioread=2.1.8=py37hc8dfbb8_2
+  - blas=1.0=mkl
+  - blinker=1.4=py37_0
+  - brotlipy=0.7.0=py37h7b6447c_1000
+  - bzip2=1.0.8=h516909a_2
+  - c-ares=1.15.0=h7b6447c_1001
+  - ca-certificates=2020.4.5.2=hecda079_0
+  - cachetools=4.1.0=py_1
+  - certifi=2020.4.5.2=py37hc8dfbb8_0
+  - cffi=1.14.0=py37he30daa8_1
+  - chardet=3.0.4=py37_1003
+  - click=7.1.2=py_0
+  - cryptography=2.9.2=py37h1ba5d50_0
+  - cycler=0.10.0=py_2
+  - decorator=4.4.2=py_0
+  - ffmpeg=4.2.3=h167e202_0
+  - freetype=2.10.2=he06d7ca_0
+  - gast=0.3.3=py_0
+  - gettext=0.19.8.1=h5e8e0c9_1
+  - gmp=6.2.0=he1b5a44_2
+  - gnutls=3.6.13=h79a8f9a_0
+  - google-auth=1.14.1=py_0
+  - google-auth-oauthlib=0.4.1=py_2
+  - google-pasta=0.2.0=py_0
+  - grpcio=1.27.2=py37hf8bcb03_0
+  - h5py=2.10.0=py37h7918eee_0
+  - hdf5=1.10.4=hb1b8bf9_0
+  - icu=58.2=hf484d3e_1000
+  - idna=2.9=py_1
+  - intel-openmp=2020.1=217
+  - joblib=0.15.1=py_0
+  - keras-preprocessing=1.1.0=py_1
+  - kiwisolver=1.2.0=py37h99015e2_0
+  - lame=3.100=h14c3975_1001
+  - ld_impl_linux-64=2.33.1=h53a641e_7
+  - libedit=3.1.20181209=hc058e9b_0
+  - libffi=3.3=he6710b0_1
+  - libflac=1.3.3=he1b5a44_0
+  - libgcc-ng=9.1.0=hdf63c60_0
+  - libgfortran-ng=7.3.0=hdf63c60_0
+  - libiconv=1.15=h516909a_1006
+  - libllvm8=8.0.1=hc9558a2_0
+  - libogg=1.3.2=h516909a_1002
+  - libpng=1.6.37=hed695b0_1
+  - libprotobuf=3.12.3=hd408876_0
+  - librosa=0.7.2=py_1
+  - libsndfile=1.0.28=he1b5a44_1000
+  - libstdcxx-ng=9.1.0=hdf63c60_0
+  - libvorbis=1.3.6=he1b5a44_2
+  - llvmlite=0.31.0=py37h5202443_1
+  - markdown=3.1.1=py37_0
+  - matplotlib-base=3.2.1=py37hef1b27d_0
+  - mkl=2020.1=217
+  - mkl-service=2.3.0=py37he904b0f_0
+  - mkl_fft=1.1.0=py37h23d657b_0
+  - mkl_random=1.1.1=py37h0573a6f_0
+  - ncurses=6.2=he6710b0_1
+  - nettle=3.4.1=h1bed415_1002
+  - numba=0.48.0=py37hb3f55d8_0
+  - numpy=1.18.1=py37h4f9e942_0
+  - numpy-base=1.18.1=py37hde5b4d6_1
+  - oauthlib=3.1.0=py_0
+  - openh264=2.1.1=h8b12597_0
+  - openssl=1.1.1g=h516909a_0
+  - opt_einsum=3.1.0=py_0
+  - pip=20.1.1=py37_1
+  - protobuf=3.12.3=py37he6710b0_0
+  - pyasn1=0.4.8=py_0
+  - pyasn1-modules=0.2.7=py_0
+  - pycparser=2.20=py_0
+  - pyjwt=1.7.1=py37_0
+  - pyopenssl=19.1.0=py37_0
+  - pyparsing=2.4.7=pyh9f0ad1d_0
+  - pysocks=1.7.1=py37_0
+  - pysoundfile=0.10.2=py_1001
+  - python=3.7.7=hcff3b4d_5
+  - python-dateutil=2.8.1=py_0
+  - python_abi=3.7=1_cp37m
+  - readline=8.0=h7b6447c_0
+  - requests=2.23.0=py37_0
+  - requests-oauthlib=1.3.0=py_0
+  - resampy=0.2.2=py_0
+  - rsa=4.0=py_0
+  - scikit-learn=0.22.1=py37hd81dba3_0
+  - scipy=1.4.1=py37h0b6359f_0
+  - setuptools=47.3.0=py37_0
+  - six=1.15.0=py_0
+  - sqlite=3.31.1=h62c20be_1
+  - tensorboard=2.2.1=pyh532a8cf_0
+  - tensorboard-plugin-wit=1.6.0=py_0
+  - tensorflow=2.2.0=mkl_py37h6e9ce2d_0
+  - tensorflow-base=2.2.0=mkl_py37hd506778_0
+  - tensorflow-estimator=2.2.0=pyh208ff02_0
+  - termcolor=1.1.0=py37_1
+  - tk=8.6.8=hbc83047_0
+  - tornado=6.0.4=py37h8f50634_1
+  - urllib3=1.25.9=py_0
+  - werkzeug=1.0.1=py_0
+  - wheel=0.34.2=py37_0
+  - wrapt=1.12.1=py37h7b6447c_1
+  - x264=1!152.20180806=h14c3975_0
+  - xz=5.2.5=h7b6447c_0
+  - zlib=1.2.11=h7b6447c_3
+  - pip:
+    - lxml==4.5.1
+    - wavinfo==1.5
+prefix: /home/nils/anaconda3/envs/eval_env
+

models/DTLN (alekya)/main.py

@@ -0,0 +1,134 @@
+# -*- coding: utf-8 -*-
+"""
+Script to process a folder of .wav files with a trained DTLN model. 
+This script supports subfolders and names the processed files the same as the 
+original. The model expects 16kHz audio .wav files. Files with other 
+sampling rates will be resampled. Stereo files will be downmixed to mono.
+
+The idea of this script is to use it for baseline or comparison purpose.
+
+Example call:
+    $python run_evaluation.py -i /name/of/input/folder  \
+                              -o /name/of/output/folder \
+                              -m /name/of/the/model.h5
+
+Author: Nils L. Westhausen (nils.westhausen@uol.de)
+Version: 13.05.2020
+
+This code is licensed under the terms of the MIT-license.
+"""
+
+import soundfile as sf
+import librosa
+import numpy as np
+import os
+import argparse
+from DTLN_model import DTLN_model
+
+
+
+def process_file(model, audio_file_name, out_file_name):
+    '''
+    Funtion to read an audio file, rocess it by the network and write the 
+    enhanced audio to .wav file.
+
+    Parameters
+    ----------
+    model : Keras model
+        Keras model, which accepts audio in the size (1,timesteps).
+    audio_file_name : STRING
+        Name and path of the input audio file.
+    out_file_name : STRING
+        Name and path of the target file.
+
+    '''
+    
+    # read audio file with librosa to handle resampling and enforce mono
+    in_data,fs = librosa.core.load(audio_file_name, sr=16000, mono=True)
+    # get length of file
+    len_orig = len(in_data)
+    # pad audio
+    zero_pad = np.zeros(384)
+    in_data = np.concatenate((zero_pad, in_data, zero_pad), axis=0)
+    # predict audio with the model
+    predicted = model.predict_on_batch(
+        np.expand_dims(in_data,axis=0).astype(np.float32))
+    # squeeze the batch dimension away
+    predicted_speech = np.squeeze(predicted)
+    predicted_speech = predicted_speech[384:384+len_orig]
+    # write the file to target destination
+    sf.write(out_file_name, predicted_speech,fs)
+      
+
+def process_folder(model, folder_name, new_folder_name):
+    '''
+    Function to find .wav files in the folder and subfolders of "folder_name",
+    process each .wav file with an algorithm and write it back to disk in the 
+    folder "new_folder_name". The structure of the original directory is 
+    preserved. The processed files will be saved with the same name as the 
+    original file.
+
+    Parameters
+    ----------
+    model : Keras model
+        Keras model, which accepts audio in the size (1,timesteps).
+    folder_name : STRING
+        Input folder with .wav files.
+    new_folder_name : STRING
+        Traget folder for the processed files.
+
+    '''
+    
+    # empty list for file and folder names
+    file_names = [];
+    directories = [];
+    new_directories = [];
+    # walk through the directory
+    for root, dirs, files in os.walk(folder_name):
+        for file in files:
+            # look for .wav files
+            if file.endswith(".wav"):
+                # write paths and filenames to lists
+                file_names.append(file)
+                directories.append(root)
+                # create new directory names
+                new_directories.append(root.replace(folder_name, new_folder_name))
+                # check if the new directory already exists, if not create it
+                if not os.path.exists(root.replace(folder_name, new_folder_name)):
+                    os.makedirs(root.replace(folder_name, new_folder_name))
+    # iterate over all .wav files             
+    for idx in range(len(file_names)):
+        # process each file with the model
+        process_file(model, os.path.join(directories[idx],file_names[idx]), 
+                     os.path.join(new_directories[idx],file_names[idx]))
+        print(file_names[idx] + ' processed successfully!')
+    
+           
+
+
+# if __name__ == '__main__':
+def run_model(model, in_folder, out_folder):
+    # arguement parser for running directly from the command line
+    # parser = argparse.ArgumentParser(description='data evaluation')
+    # parser.add_argument('--in_folder', '-i',
+    #                     help='folder with input files')
+    # parser.add_argument('--out_folder', '-o',
+    #                     help='target folder for processed files')
+    # parser.add_argument('--model', '-m',
+    #                     help='weights of the enhancement model in .h5 format')
+    # args = parser.parse_args()
+    # # determine type of model
+    if model.find('_norm_') != -1:
+        norm_stft = True
+    else:
+        norm_stft = False
+    # create class instance
+    modelClass = DTLN_model();
+    # build the model in default configuration
+    modelClass.build_DTLN_model(norm_stft=norm_stft)
+    # load weights of the .h5 file
+    modelClass.model.load_weights(model)
+    # process the folder
+    process_folder(modelClass.model, in_folder, out_folder)
+
+

models/DTLN (alekya)/main_1.py

@@ -0,0 +1,3 @@
+from main import run_model
+
+run_model("./pretrained_model/model.h5", "test", "./output_test_2")

models/DTLN (alekya)/measure_execution_time.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+This script tests the execution time of the DTLN model on a CPU.
+Please use TF 2.2 for comparability.
+
+Just run "python measure_execution_time.py"
+
+Author: Nils L. Westhausen (nils.westhausen@uol.de)
+Version: 13.05.2020
+
+This code is licensed under the terms of the MIT-license.
+"""
+
+import time
+import tensorflow as tf 
+import numpy as np
+import os
+
+# only use the cpu
+os.environ["CUDA_VISIBLE_DEVICES"]=''
+
+if __name__ == '__main__':
+    # loading model in saved model format
+    model = tf.saved_model.load('./pretrained_model/dtln_saved_model')
+    # mapping signature names to functions
+    infer = model.signatures["serving_default"]
+        
+    exec_time = []
+    # create random input for testing
+    x = np.random.randn(1,512).astype('float32')
+    for idx in range(1010):
+        # run timer
+        start_time = time.time()
+        # infer one block
+        y = infer(tf.constant(x))['conv1d_1']
+        exec_time.append((time.time() - start_time))
+    # ignore the first ten iterations
+    print('Execution time per block: ' + 
+          str( np.round(np.mean(np.stack(exec_time[10:]))*1000, 2)) + ' ms')
+
+# Ubuntu 18.04          I5 6600k        @ 3.5 GHz:  0.65 ms (4 cores)
+# Macbook Air mid 2012 	I7 3667U        @ 2.0 GHz:  1.4 ms  (2 cores)
+# Raspberry Pi 3 B+     ARM Cortex A53  @ 1.4 GHz: 15.54    (4 cores)

models/DTLN (alekya)/model.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6e9ecb8140f51e8ffb373884b62e186dc166e97995fa7eea3fffcc6ef30d55d
+size 3989312

models/DTLN (alekya)/real_time_dtln_audio.py

@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+This is a real time example how to implement DTLN tf light model with 
+sounddevice. The script is based on the "wire.py" example of the sounddevice 
+toolbox. If the command line shows "input underflow", restart the script.
+If there are still a lot of dropouts, increase the latency.
+
+First call:
+    
+    $ python real_time_dtln_audio.py --list-devices
+
+to get your audio devices. In the next step call
+
+    $ python real_time_dtln_audio.py -i in_device_idx -o out_device_idx
+
+For .whl files of the tf light runtime go to: 
+    https://www.tensorflow.org/lite/guide/python
+    
+    
+
+Author: Nils L. Westhausen (nils.westhausen@uol.de)
+Version: 01.07.2020
+
+This code is licensed under the terms of the MIT-license.
+"""
+
+
+import numpy as np
+import sounddevice as sd
+import tflite_runtime.interpreter as tflite
+import argparse
+
+
+def int_or_str(text):
+    """Helper function for argument parsing."""
+    try:
+        return int(text)
+    except ValueError:
+        return text
+
+
+parser = argparse.ArgumentParser(add_help=False)
+parser.add_argument(
+    '-l', '--list-devices', action='store_true',
+    help='show list of audio devices and exit')
+args, remaining = parser.parse_known_args()
+if args.list_devices:
+    print(sd.query_devices())
+    parser.exit(0)
+parser = argparse.ArgumentParser(
+    description=__doc__,
+    formatter_class=argparse.RawDescriptionHelpFormatter,
+    parents=[parser])
+parser.add_argument(
+    '-i', '--input-device', type=int_or_str,
+    help='input device (numeric ID or substring)')
+parser.add_argument(
+    '-o', '--output-device', type=int_or_str,
+    help='output device (numeric ID or substring)')
+
+parser.add_argument('--latency', type=float, help='latency in seconds', default=0.2)
+args = parser.parse_args(remaining)
+
+# set some parameters
+block_len_ms = 32 
+block_shift_ms = 8
+fs_target = 16000
+# create the interpreters
+interpreter_1 = tflite.Interpreter(model_path='./pretrained_model/model_1.tflite')
+interpreter_1.allocate_tensors()
+interpreter_2 = tflite.Interpreter(model_path='./pretrained_model/model_2.tflite')
+interpreter_2.allocate_tensors()
+# Get input and output tensors.
+input_details_1 = interpreter_1.get_input_details()
+output_details_1 = interpreter_1.get_output_details()
+input_details_2 = interpreter_2.get_input_details()
+output_details_2 = interpreter_2.get_output_details()
+# create states for the lstms
+states_1 = np.zeros(input_details_1[1]['shape']).astype('float32')
+states_2 = np.zeros(input_details_2[1]['shape']).astype('float32')
+# calculate shift and length
+block_shift = int(np.round(fs_target * (block_shift_ms / 1000)))
+block_len = int(np.round(fs_target * (block_len_ms / 1000)))
+# create buffer
+in_buffer = np.zeros((block_len)).astype('float32')
+out_buffer = np.zeros((block_len)).astype('float32')
+
+
+def callback(indata, outdata, frames, time, status):
+    # buffer and states to global
+    global in_buffer, out_buffer, states_1, states_2
+    if status:
+        print(status)
+    # write to buffer
+    in_buffer[:-block_shift] = in_buffer[block_shift:]
+    in_buffer[-block_shift:] = np.squeeze(indata)
+    # calculate fft of input block
+    in_block_fft = np.fft.rfft(in_buffer)
+    in_mag = np.abs(in_block_fft)
+    in_phase = np.angle(in_block_fft)
+    # reshape magnitude to input dimensions
+    in_mag = np.reshape(in_mag, (1,1,-1)).astype('float32')
+    # set tensors to the first model
+    interpreter_1.set_tensor(input_details_1[1]['index'], states_1)
+    interpreter_1.set_tensor(input_details_1[0]['index'], in_mag)
+    # run calculation 
+    interpreter_1.invoke()
+    # get the output of the first block
+    out_mask = interpreter_1.get_tensor(output_details_1[0]['index']) 
+    states_1 = interpreter_1.get_tensor(output_details_1[1]['index'])   
+    # calculate the ifft
+    estimated_complex = in_mag * out_mask * np.exp(1j * in_phase)
+    estimated_block = np.fft.irfft(estimated_complex)
+    # reshape the time domain block
+    estimated_block = np.reshape(estimated_block, (1,1,-1)).astype('float32')
+    # set tensors to the second block
+    interpreter_2.set_tensor(input_details_2[1]['index'], states_2)
+    interpreter_2.set_tensor(input_details_2[0]['index'], estimated_block)
+    # run calculation
+    interpreter_2.invoke()
+    # get output tensors
+    out_block = interpreter_2.get_tensor(output_details_2[0]['index']) 
+    states_2 = interpreter_2.get_tensor(output_details_2[1]['index']) 
+    # write to buffer
+    out_buffer[:-block_shift] = out_buffer[block_shift:]
+    out_buffer[-block_shift:] = np.zeros((block_shift))
+    out_buffer  += np.squeeze(out_block)
+    # output to soundcard
+    outdata[:] = np.expand_dims(out_buffer[:block_shift], axis=-1)
+    
+
+
+try:
+    with sd.Stream(device=(args.input_device, args.output_device),
+                   samplerate=fs_target, blocksize=block_shift,
+                   dtype=np.float32, latency=args.latency,
+                   channels=1, callback=callback):
+        print('#' * 80)
+        print('press Return to quit')
+        print('#' * 80)
+        input()
+except KeyboardInterrupt:
+    parser.exit('')
+except Exception as e:
+    parser.exit(type(e).__name__ + ': ' + str(e))
+    

models/DTLN (alekya)/real_time_processing.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Tue Jun 23 16:23:15 2020
+
+@author: nils
+"""
+
+import soundfile as sf
+import numpy as np
+import tensorflow as tf
+
+
+
+##########################
+# the values are fixed, if you need other values, you have to retrain.
+# The sampling rate of 16k is also fix.
+block_len = 512
+block_shift = 128
+# load model
+model = tf.saved_model.load('./pretrained_model/dtln_saved_model')
+infer = model.signatures["serving_default"]
+# load audio file at 16k fs (please change)
+audio,fs = sf.read('path_to_your_favorite_audio.wav')
+# check for sampling rate
+if fs != 16000:
+    raise ValueError('This model only supports 16k sampling rate.')
+# preallocate output audio
+out_file = np.zeros((len(audio)))
+# create buffer
+in_buffer = np.zeros((block_len))
+out_buffer = np.zeros((block_len))
+# calculate number of blocks
+num_blocks = (audio.shape[0] - (block_len-block_shift)) // block_shift
+# iterate over the number of blcoks        
+for idx in range(num_blocks):
+    # shift values and write to buffer
+    in_buffer[:-block_shift] = in_buffer[block_shift:]
+    in_buffer[-block_shift:] = audio[idx*block_shift:(idx*block_shift)+block_shift]
+    # create a batch dimension of one
+    in_block = np.expand_dims(in_buffer, axis=0).astype('float32')
+    # process one block
+    out_block= infer(tf.constant(in_block))['conv1d_1']
+    # shift values and write to buffer
+    out_buffer[:-block_shift] = out_buffer[block_shift:]
+    out_buffer[-block_shift:] = np.zeros((block_shift))
+    out_buffer  += np.squeeze(out_block)
+    # write block to output file
+    out_file[idx*block_shift:(idx*block_shift)+block_shift] = out_buffer[:block_shift]
+    
+    
+# write to .wav file 
+sf.write('out.wav', out_file, fs) 
+
+print('Processing finished.')

models/DTLN (alekya)/real_time_processing_onnx.py

@@ -0,0 +1,106 @@
+"""
+This is an example how to implement real time processing of the DTLN ONNX
+model in python.
+
+Please change the name of the .wav file at line 49 before running the sript.
+For the ONNX runtime call: $ pip install onnxruntime
+    
+    
+
+Author: Nils L. Westhausen (nils.westhausen@uol.de)
+Version: 03.07.2020
+
+This code is licensed under the terms of the MIT-license.
+"""
+
+import soundfile as sf
+import numpy as np
+import time
+import onnxruntime
+
+
+
+##########################
+# the values are fixed, if you need other values, you have to retrain.
+# The sampling rate of 16k is also fix.
+block_len = 512
+block_shift = 128
+# load models
+interpreter_1 = onnxruntime.InferenceSession('./model_1.onnx')
+model_input_names_1 = [inp.name for inp in interpreter_1.get_inputs()]
+# preallocate input
+model_inputs_1 = {
+            inp.name: np.zeros(
+                [dim if isinstance(dim, int) else 1 for dim in inp.shape],
+                dtype=np.float32)
+            for inp in interpreter_1.get_inputs()}
+# load models
+interpreter_2 = onnxruntime.InferenceSession('./model_2.onnx')
+model_input_names_2 = [inp.name for inp in interpreter_2.get_inputs()]
+# preallocate input
+model_inputs_2 = {
+            inp.name: np.zeros(
+                [dim if isinstance(dim, int) else 1 for dim in inp.shape],
+                dtype=np.float32)
+            for inp in interpreter_2.get_inputs()}
+
+# load audio file
+audio,fs = sf.read('path/to/your/favorite.wav')
+# check for sampling rate
+if fs != 16000:
+    raise ValueError('This model only supports 16k sampling rate.')
+# preallocate output audio
+out_file = np.zeros((len(audio)))
+# create buffer
+in_buffer = np.zeros((block_len)).astype('float32')
+out_buffer = np.zeros((block_len)).astype('float32')
+# calculate number of blocks
+num_blocks = (audio.shape[0] - (block_len-block_shift)) // block_shift
+# iterate over the number of blcoks  
+time_array = []      
+for idx in range(num_blocks):
+    start_time = time.time()
+    # shift values and write to buffer
+    in_buffer[:-block_shift] = in_buffer[block_shift:]
+    in_buffer[-block_shift:] = audio[idx*block_shift:(idx*block_shift)+block_shift]
+    # calculate fft of input block
+    in_block_fft = np.fft.rfft(in_buffer)
+    in_mag = np.abs(in_block_fft)
+    in_phase = np.angle(in_block_fft)
+    # reshape magnitude to input dimensions
+    in_mag = np.reshape(in_mag, (1,1,-1)).astype('float32')
+    # set block to input
+    model_inputs_1[model_input_names_1[0]] = in_mag
+    # run calculation 
+    model_outputs_1 = interpreter_1.run(None, model_inputs_1)
+    # get the output of the first block
+    out_mask = model_outputs_1[0]
+    # set out states back to input
+    model_inputs_1[model_input_names_1[1]] = model_outputs_1[1]  
+    # calculate the ifft
+    estimated_complex = in_mag * out_mask * np.exp(1j * in_phase)
+    estimated_block = np.fft.irfft(estimated_complex)
+    # reshape the time domain block
+    estimated_block = np.reshape(estimated_block, (1,1,-1)).astype('float32')
+    # set tensors to the second block
+    # interpreter_2.set_tensor(input_details_1[1]['index'], states_2)
+    model_inputs_2[model_input_names_2[0]] = estimated_block
+    # run calculation
+    model_outputs_2 = interpreter_2.run(None, model_inputs_2)
+    # get output
+    out_block = model_outputs_2[0]
+    # set out states back to input
+    model_inputs_2[model_input_names_2[1]] = model_outputs_2[1]
+    # shift values and write to buffer
+    out_buffer[:-block_shift] = out_buffer[block_shift:]
+    out_buffer[-block_shift:] = np.zeros((block_shift))
+    out_buffer  += np.squeeze(out_block)
+    # write block to output file
+    out_file[idx*block_shift:(idx*block_shift)+block_shift] = out_buffer[:block_shift]
+    time_array.append(time.time()-start_time)
+    
+# write to .wav file 
+sf.write('out.wav', out_file, fs) 
+print('Processing Time [ms]:')
+print(np.mean(np.stack(time_array))*1000)
+print('Processing finished.')

models/DTLN (alekya)/real_time_processing_tf_lite.py

@@ -0,0 +1,102 @@
+"""
+This is an example how to implement real time processing of the DTLN tf light
+model in python.
+
+Please change the name of the .wav file at line 43 before running the sript.
+For .whl files of the tf light runtime go to: 
+    https://www.tensorflow.org/lite/guide/python
+    
+Author: Nils L. Westhausen (nils.westhausen@uol.de)
+Version: 30.06.2020
+
+This code is licensed under the terms of the MIT-license.
+"""
+
+import soundfile as sf
+import numpy as np
+import tflite_runtime.interpreter as tflite
+import time
+
+
+
+##########################
+# the values are fixed, if you need other values, you have to retrain.
+# The sampling rate of 16k is also fix.
+block_len = 512
+block_shift = 128
+# load models
+interpreter_1 = tflite.Interpreter(model_path='./pretrained_model/model_1.tflite')
+interpreter_1.allocate_tensors()
+interpreter_2 = tflite.Interpreter(model_path='./pretrained_model/model_2.tflite')
+interpreter_2.allocate_tensors()
+
+# Get input and output tensors.
+input_details_1 = interpreter_1.get_input_details()
+output_details_1 = interpreter_1.get_output_details()
+
+input_details_2 = interpreter_2.get_input_details()
+output_details_2 = interpreter_2.get_output_details()
+# create states for the lstms
+states_1 = np.zeros(input_details_1[1]['shape']).astype('float32')
+states_2 = np.zeros(input_details_2[1]['shape']).astype('float32')
+# load audio file at 16k fs (please change)
+audio,fs = sf.read('path/to/your/favorite/.wav')
+# check for sampling rate
+if fs != 16000:
+    raise ValueError('This model only supports 16k sampling rate.')
+# preallocate output audio
+out_file = np.zeros((len(audio)))
+# create buffer
+in_buffer = np.zeros((block_len)).astype('float32')
+out_buffer = np.zeros((block_len)).astype('float32')
+# calculate number of blocks
+num_blocks = (audio.shape[0] - (block_len-block_shift)) // block_shift
+time_array = []      
+# iterate over the number of blcoks  
+for idx in range(num_blocks):
+    start_time = time.time()
+    # shift values and write to buffer
+    in_buffer[:-block_shift] = in_buffer[block_shift:]
+    in_buffer[-block_shift:] = audio[idx*block_shift:(idx*block_shift)+block_shift]
+    # calculate fft of input block
+    in_block_fft = np.fft.rfft(in_buffer)
+    in_mag = np.abs(in_block_fft)
+    in_phase = np.angle(in_block_fft)
+    # reshape magnitude to input dimensions
+    in_mag = np.reshape(in_mag, (1,1,-1)).astype('float32')
+    # set tensors to the first model
+    interpreter_1.set_tensor(input_details_1[1]['index'], states_1)
+    interpreter_1.set_tensor(input_details_1[0]['index'], in_mag)
+    # run calculation 
+    interpreter_1.invoke()
+    # get the output of the first block
+    out_mask = interpreter_1.get_tensor(output_details_1[0]['index']) 
+    states_1 = interpreter_1.get_tensor(output_details_1[1]['index'])   
+    # calculate the ifft
+    estimated_complex = in_mag * out_mask * np.exp(1j * in_phase)
+    estimated_block = np.fft.irfft(estimated_complex)
+    # reshape the time domain block
+    estimated_block = np.reshape(estimated_block, (1,1,-1)).astype('float32')
+    # set tensors to the second block
+    interpreter_2.set_tensor(input_details_2[1]['index'], states_2)
+    interpreter_2.set_tensor(input_details_2[0]['index'], estimated_block)
+    # run calculation
+    interpreter_2.invoke()
+    # get output tensors
+    out_block = interpreter_2.get_tensor(output_details_2[0]['index']) 
+    states_2 = interpreter_2.get_tensor(output_details_2[1]['index']) 
+    
+    
+    # shift values and write to buffer
+    out_buffer[:-block_shift] = out_buffer[block_shift:]
+    out_buffer[-block_shift:] = np.zeros((block_shift))
+    out_buffer  += np.squeeze(out_block)
+    # write block to output file
+    out_file[idx*block_shift:(idx*block_shift)+block_shift] = out_buffer[:block_shift]
+    time_array.append(time.time()-start_time)
+    
+# write to .wav file 
+sf.write('out.wav', out_file, fs) 
+print('Processing Time [ms]:')
+print(np.mean(np.stack(time_array))*1000)
+print('Processing finished.')

models/DTLN (alekya)/run_evaluation.py

@@ -0,0 +1,131 @@
+# -*- coding: utf-8 -*-
+"""
+Script to process a folder of .wav files with a trained DTLN model. 
+This script supports subfolders and names the processed files the same as the 
+original. The model expects 16kHz audio .wav files. Files with other 
+sampling rates will be resampled. Stereo files will be downmixed to mono.
+
+The idea of this script is to use it for baseline or comparison purpose.
+
+Example call:
+    $python run_evaluation.py -i /name/of/input/folder  \
+                              -o /name/of/output/folder \
+                              -m /name/of/the/model.h5
+
+Author: Nils L. Westhausen (nils.westhausen@uol.de)
+Version: 13.05.2020
+
+This code is licensed under the terms of the MIT-license.
+"""
+
+import soundfile as sf
+import librosa
+import numpy as np
+import os
+import argparse
+from DTLN_model import DTLN_model
+
+
+
+def process_file(model, audio_file_name, out_file_name):
+    '''
+    Funtion to read an audio file, rocess it by the network and write the 
+    enhanced audio to .wav file.
+
+    Parameters
+    ----------
+    model : Keras model
+        Keras model, which accepts audio in the size (1,timesteps).
+    audio_file_name : STRING
+        Name and path of the input audio file.
+    out_file_name : STRING
+        Name and path of the target file.
+
+    '''
+    
+    # read audio file with librosa to handle resampling and enforce mono
+    in_data,fs = librosa.core.load(audio_file_name, sr=16000, mono=True)
+    # get length of file
+    len_orig = len(in_data)
+    # pad audio
+    zero_pad = np.zeros(384)
+    in_data = np.concatenate((zero_pad, in_data, zero_pad), axis=0)
+    # predict audio with the model
+    predicted = model.predict_on_batch(
+        np.expand_dims(in_data,axis=0).astype(np.float32))
+    # squeeze the batch dimension away
+    predicted_speech = np.squeeze(predicted)
+    predicted_speech = predicted_speech[384:384+len_orig]
+    # write the file to target destination
+    sf.write(out_file_name, predicted_speech,fs)
+      
+
+def process_folder(model, folder_name, new_folder_name):
+    '''
+    Function to find .wav files in the folder and subfolders of "folder_name",
+    process each .wav file with an algorithm and write it back to disk in the 
+    folder "new_folder_name". The structure of the original directory is 
+    preserved. The processed files will be saved with the same name as the 
+    original file.
+
+    Parameters
+    ----------
+    model : Keras model
+        Keras model, which accepts audio in the size (1,timesteps).
+    folder_name : STRING
+        Input folder with .wav files.
+    new_folder_name : STRING
+        Traget folder for the processed files.
+
+    '''
+    
+    # empty list for file and folder names
+    file_names = [];
+    directories = [];
+    new_directories = [];
+    # walk through the directory
+    for root, dirs, files in os.walk(folder_name):
+        for file in files:
+            # look for .wav files
+            if file.endswith(".wav"):
+                # write paths and filenames to lists
+                file_names.append(file)
+                directories.append(root)
+                # create new directory names
+                new_directories.append(root.replace(folder_name, new_folder_name))
+                # check if the new directory already exists, if not create it
+                if not os.path.exists(root.replace(folder_name, new_folder_name)):
+                    os.makedirs(root.replace(folder_name, new_folder_name))
+    # iterate over all .wav files             
+    for idx in range(len(file_names)):
+        # process each file with the model
+        process_file(model, os.path.join(directories[idx],file_names[idx]), 
+                     os.path.join(new_directories[idx],file_names[idx]))
+        print(file_names[idx] + ' processed successfully!')
+    
+           
+
+
+if __name__ == '__main__':
+    # arguement parser for running directly from the command line
+    parser = argparse.ArgumentParser(description='data evaluation')
+    parser.add_argument('--in_folder', '-i',
+                        help='folder with input files')
+    parser.add_argument('--out_folder', '-o',
+                        help='target folder for processed files')
+    parser.add_argument('--model', '-m',
+                        help='weights of the enhancement model in .h5 format')
+    args = parser.parse_args()
+    # determine type of model
+    if args.model.find('_norm_') != -1:
+        norm_stft = True
+    else:
+        norm_stft = False
+    # create class instance
+    modelClass = DTLN_model();
+    # build the model in default configuration
+    modelClass.build_DTLN_model(norm_stft=norm_stft)
+    # load weights of the .h5 file
+    modelClass.model.load_weights(args.model)
+    # process the folder
+    process_folder(modelClass.model, args.in_folder, args.out_folder)

models/DTLN (alekya)/run_training.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Script to train the DTLN model in default settings. The folders for noisy and
+clean files are expected to have the same number of files and the files to 
+have the same name. The training procedure always saves the best weights of 
+the model into the folder "./models_'runName'/". Also a log file of the 
+training progress is written there. To change any parameters go to the 
+"DTLN_model.py" file or use "modelTrainer.parameter = XY" in this file.
+It is recommended to run the training on a GPU. The setup is optimized for the
+DNS-Challenge data set. If you use a custom data set, just play around with
+the parameters.
+
+Please change the folder names before starting the training. 
+
+Example call:
+    $python run_training.py
+
+Author: Nils L. Westhausen (nils.westhausen@uol.de)
+Version: 13.05.2020
+
+This code is licensed under the terms of the MIT-license.
+"""
+
+from DTLN_model import DTLN_model
+import os
+
+# use the GPU with idx 0
+os.environ["CUDA_VISIBLE_DEVICES"]='0'
+# activate this for some reproducibility
+os.environ['TF_DETERMINISTIC_OPS'] = '1'
+
+
+
+
+
+# path to folder containing the noisy or mixed audio training files
+path_to_train_mix = '/path/to/noisy/training/data/'
+# path to folder containing the clean/speech files for training
+path_to_train_speech = '/path/to/clean/training/data/'
+# path to folder containing the noisy or mixed audio validation data
+path_to_val_mix = '/path/to/noisy/validation/data/'
+# path to folder containing the clean audio validation data
+path_to_val_speech = '/path/to/clean/validation/data/'
+
+# name your training run
+runName = 'DTLN_model'
+# create instance of the DTLN model class
+modelTrainer = DTLN_model()
+# build the model
+modelTrainer.build_DTLN_model()
+# compile it with optimizer and cost function for training
+modelTrainer.compile_model()
+# train the model
+modelTrainer.train_model(runName, path_to_train_mix, path_to_train_speech, \
+                         path_to_val_mix, path_to_val_speech)
+
+
+

models/DTLN (alekya)/source.txt

@@ -0,0 +1 @@
+https://huggingface.co/alekya/DTLN

models/DTLN (alekya)/test.py

@@ -0,0 +1,47 @@
+import IPython
+import numpy as np
+import matplotlib.pyplot as plt
+from scipy.io import wavfile
+import wave as we
+import soundfile as sf
+import os
+import librosa
+from pesq import pesq
+
+clean_path = "/home/ap/Desktop/Workspace/DTLN/valset_clean/p232_007.wav"
+denoised_path = '/home/ap/Desktop/Workspace/DTLN/output-denoised-audio (1).wav'
+
+# denoised_files = os.listdir(denoised_path)
+# clean_files = os.listdir(clean_path)
+
+# rmse = []
+# snr_list = []
+# pesq_list = []
+
+# for i in range(len(denoised_files)):
+original, sr1 = librosa.load(clean_path, sr=None)
+denoised, sr2 = librosa.load(denoised_path, sr=None)
+
+
+min_length = min(len(original), len(denoised))
+original = original[:min_length]
+denoised = denoised[:min_length]
+
+mse = np.mean((original - denoised) ** 2)
+
+# Calculate Signal-to-Noise Ratio (SNR)
+signal_power = np.mean(original ** 2)
+noise_power = np.mean((original - denoised) ** 2)
+snr = 10 * np.log10(signal_power / noise_power)
+
+# rmse.append(mse)
+# snr_list.append(snr)
+
+
+print(f"Mean Squared Error: {np.average(mse)}")
+print(f"Signal-to-Noise Ratio (SNR): {np.average(snr)} dB")
+
+rate, ref = wavfile.read("/home/ap/Desktop/Workspace/DTLN/test/19-198-0034.wav")
+rate, deg = wavfile.read("/home/ap/Desktop/Workspace/DTLN/output_test/19-198-0034.wav")
+
+print(pesq(rate, ref, deg, 'wb'))

models/DTLN (alekya)/tflite_env.yml

@@ -0,0 +1,73 @@
+name: tflite-env
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - audioread=2.1.8=py37hc8dfbb8_2
+  - bzip2=1.0.8=h0b31af3_2
+  - ca-certificates=2020.6.20=hecda079_0
+  - certifi=2020.6.20=py37hc8dfbb8_0
+  - cffi=1.14.0=py37hc512035_1
+  - cycler=0.10.0=py_2
+  - decorator=4.4.2=py_0
+  - ffmpeg=4.2.3=hd0c0d6a_0
+  - freetype=2.10.2=h8da9a1a_0
+  - gettext=0.19.8.1=h1f1d5ed_1
+  - gmp=6.2.0=h4a8c4bd_2
+  - gnutls=3.6.13=hc269f14_0
+  - joblib=0.15.1=py_0
+  - kiwisolver=1.2.0=py37ha1cc60f_0
+  - lame=3.100=h1de35cc_1001
+  - libblas=3.8.0=17_openblas
+  - libcblas=3.8.0=17_openblas
+  - libcxx=10.0.0=1
+  - libedit=3.1.20191231=haf1e3a3_0
+  - libffi=3.3=h0a44026_1
+  - libflac=1.3.3=h4a8c4bd_0
+  - libgfortran=4.0.0=2
+  - libiconv=1.15=h0b31af3_1006
+  - liblapack=3.8.0=17_openblas
+  - libllvm8=8.0.1=h770b8ee_0
+  - libogg=1.3.2=h0b31af3_1002
+  - libopenblas=0.3.10=h3d69b6c_0
+  - libpng=1.6.37=hbbe82c9_1
+  - librosa=0.7.2=py_1
+  - libsndfile=1.0.28=h4a8c4bd_1000
+  - libvorbis=1.3.6=h4a8c4bd_2
+  - llvm-openmp=10.0.0=h28b9765_0
+  - llvmlite=0.31.0=py37hb548287_1
+  - matplotlib-base=3.2.2=py37hddda452_0
+  - ncurses=6.2=h0a44026_1
+  - nettle=3.4.1=h3efe00b_1002
+  - numba=0.48.0=py37h4f17bb1_0
+  - numpy=1.18.5=py37h7687784_0
+  - openh264=2.1.1=hd174df1_0
+  - openssl=1.1.1g=h0b31af3_0
+  - pip=20.1.1=py37_1
+  - portaudio=19.6.0=h647c56a_4
+  - pycparser=2.20=pyh9f0ad1d_2
+  - pyparsing=2.4.7=pyh9f0ad1d_0
+  - pysoundfile=0.10.2=py_1001
+  - python=3.7.7=hf48f09d_4
+  - python-dateutil=2.8.1=py_0
+  - python-sounddevice=0.3.15=pyh8c360ce_0
+  - python_abi=3.7=1_cp37m
+  - readline=8.0=h1de35cc_0
+  - resampy=0.2.2=py_0
+  - scikit-learn=0.23.1=py37hf5857e7_0
+  - scipy=1.5.0=py37hce1b9e5_0
+  - setuptools=47.3.1=py37_0
+  - six=1.15.0=pyh9f0ad1d_0
+  - sqlite=3.32.3=hffcf06c_0
+  - threadpoolctl=2.1.0=pyh5ca1d4c_0
+  - tk=8.6.10=hb0a8c7a_0
+  - tornado=6.0.4=py37h9bfed18_1
+  - wheel=0.34.2=py37_0
+  - x264=1!152.20180806=h1de35cc_0
+  - xz=5.2.5=h1de35cc_0
+  - zlib=1.2.11=h1de35cc_3
+  - pip:
+    - flatbuffers==1.12
+    - tflite-runtime==2.1.0.post1
+prefix: /Applications/anaconda3/envs/tflite-env
+

models/DTLN (alekya)/train_env.yml

@@ -0,0 +1,120 @@
+name: train_env
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=main
+  - _tflow_select=2.1.0=gpu
+  - absl-py=0.9.0=py37_0
+  - astunparse=1.6.3=py_0
+  - audioread=2.1.8=py37hc8dfbb8_2
+  - blas=1.0=mkl
+  - blinker=1.4=py37_0
+  - bzip2=1.0.8=h516909a_2
+  - c-ares=1.15.0=h7b6447c_1001
+  - ca-certificates=2020.4.5.2=hecda079_0
+  - cachetools=4.1.0=py_1
+  - certifi=2020.4.5.2=py37hc8dfbb8_0
+  - cffi=1.14.0=py37he30daa8_1
+  - chardet=3.0.4=py37_1003
+  - click=7.1.2=py_0
+  - cryptography=2.9.2=py37h1ba5d50_0
+  - cudatoolkit=10.1.243=h6bb024c_0
+  - cudnn=7.6.5=cuda10.1_0
+  - cupti=10.1.168=0
+  - cycler=0.10.0=py_2
+  - decorator=4.4.2=py_0
+  - ffmpeg=4.2.3=h167e202_0
+  - freetype=2.10.2=he06d7ca_0
+  - gast=0.3.3=py_0
+  - gettext=0.19.8.1=h5e8e0c9_1
+  - gmp=6.2.0=he1b5a44_2
+  - gnutls=3.6.13=h79a8f9a_0
+  - google-auth=1.14.1=py_0
+  - google-auth-oauthlib=0.4.1=py_2
+  - google-pasta=0.2.0=py_0
+  - grpcio=1.27.2=py37hf8bcb03_0
+  - h5py=2.10.0=py37h7918eee_0
+  - hdf5=1.10.4=hb1b8bf9_0
+  - icu=58.2=hf484d3e_1000
+  - idna=2.9=py_1
+  - intel-openmp=2020.1=217
+  - joblib=0.15.1=py_0
+  - keras-preprocessing=1.1.0=py_1
+  - kiwisolver=1.2.0=py37h99015e2_0
+  - lame=3.100=h14c3975_1001
+  - ld_impl_linux-64=2.33.1=h53a641e_7
+  - libedit=3.1.20181209=hc058e9b_0
+  - libffi=3.3=he6710b0_1
+  - libflac=1.3.3=he1b5a44_0
+  - libgcc-ng=9.1.0=hdf63c60_0
+  - libgfortran-ng=7.3.0=hdf63c60_0
+  - libiconv=1.15=h516909a_1006
+  - libllvm8=8.0.1=hc9558a2_0
+  - libogg=1.3.2=h516909a_1002
+  - libpng=1.6.37=hed695b0_1
+  - libprotobuf=3.12.3=hd408876_0
+  - librosa=0.7.2=py_1
+  - libsndfile=1.0.28=he1b5a44_1000
+  - libstdcxx-ng=9.1.0=hdf63c60_0
+  - libvorbis=1.3.6=he1b5a44_2
+  - llvmlite=0.31.0=py37h5202443_1
+  - markdown=3.1.1=py37_0
+  - matplotlib-base=3.1.3=py37hef1b27d_0
+  - mkl=2020.1=217
+  - mkl-service=2.3.0=py37he904b0f_0
+  - mkl_fft=1.0.15=py37ha843d7b_0
+  - mkl_random=1.1.1=py37h0573a6f_0
+  - ncurses=6.2=he6710b0_1
+  - nettle=3.4.1=h1bed415_1002
+  - numba=0.48.0=py37hb3f55d8_0
+  - numpy=1.18.1=py37h4f9e942_0
+  - numpy-base=1.18.1=py37hde5b4d6_1
+  - oauthlib=3.1.0=py_0
+  - openh264=2.1.1=h8b12597_0
+  - openssl=1.1.1g=h516909a_0
+  - opt_einsum=3.1.0=py_0
+  - pip=20.0.2=py37_3
+  - protobuf=3.12.3=py37he6710b0_0
+  - pyasn1=0.4.8=py_0
+  - pyasn1-modules=0.2.7=py_0
+  - pycparser=2.20=py_0
+  - pyjwt=1.7.1=py37_0
+  - pyopenssl=19.1.0=py37_0
+  - pyparsing=2.4.7=pyh9f0ad1d_0
+  - pysocks=1.7.1=py37_0
+  - pysoundfile=0.10.2=py_1001
+  - python=3.7.7=hcff3b4d_5
+  - python-dateutil=2.8.1=py_0
+  - python_abi=3.7=1_cp37m
+  - readline=8.0=h7b6447c_0
+  - requests=2.23.0=py37_0
+  - requests-oauthlib=1.3.0=py_0
+  - resampy=0.2.2=py_0
+  - rsa=4.0=py_0
+  - scikit-learn=0.22.1=py37hd81dba3_0
+  - scipy=1.4.1=py37h0b6359f_0
+  - setuptools=47.1.1=py37_0
+  - six=1.15.0=py_0
+  - sqlite=3.31.1=h62c20be_1
+  - tensorboard=2.2.1=pyh532a8cf_0
+  - tensorboard-plugin-wit=1.6.0=py_0
+  - tensorflow=2.2.0=gpu_py37h1a511ff_0
+  - tensorflow-base=2.2.0=gpu_py37h8a81be8_0
+  - tensorflow-estimator=2.2.0=pyh208ff02_0
+  - tensorflow-gpu=2.2.0=h0d30ee6_0
+  - termcolor=1.1.0=py37_1
+  - tk=8.6.8=hbc83047_0
+  - tornado=6.0.4=py37h8f50634_1
+  - urllib3=1.25.8=py37_0
+  - werkzeug=1.0.1=py_0
+  - wheel=0.34.2=py37_0
+  - wrapt=1.12.1=py37h7b6447c_1
+  - x264=1!152.20180806=h14c3975_0
+  - xz=5.2.5=h7b6447c_0
+  - zlib=1.2.11=h7b6447c_3
+  - pip:
+    - lxml==4.5.1
+    - wavinfo==1.5
+prefix: /home/nils/anaconda3/envs/tfenv
+

models/DTLN (yash-04)/.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

models/DTLN (yash-04)/DTLN_model.py

@@ -0,0 +1,641 @@
+# -*- coding: utf-8 -*-
+"""
+This File contains everything to train the DTLN model.
+
+For running the training see "run_training.py".
+To run evaluation with the provided pretrained model see "run_evaluation.py".
+
+Author: Nils L. Westhausen (nils.westhausen@uol.de)
+Version: 24.06.2020
+
+This code is licensed under the terms of the MIT-license.
+"""
+
+
+import os, fnmatch
+import tensorflow.keras as keras
+from tensorflow.keras.models import Model
+from tensorflow.keras.layers import Activation, Dense, LSTM, Dropout, \
+    Lambda, Input, Multiply, Layer, Conv1D
+from tensorflow.keras.callbacks import ReduceLROnPlateau, CSVLogger, \
+    EarlyStopping, ModelCheckpoint
+import tensorflow as tf
+import soundfile as sf
+from wavinfo import WavInfoReader
+from random import shuffle, seed
+import numpy as np
+
+
+
+class audio_generator():
+    '''
+    Class to create a Tensorflow dataset based on an iterator from a large scale 
+    audio dataset. This audio generator only supports single channel audio files.
+    '''
+    
+    def __init__(self, path_to_input, path_to_s1, len_of_samples, fs, train_flag=False):
+        '''
+        Constructor of the audio generator class.
+        Inputs:
+            path_to_input       path to the mixtures
+            path_to_s1          path to the target source data
+            len_of_samples      length of audio snippets in samples
+            fs                  sampling rate
+            train_flag          flag for activate shuffling of files
+        '''
+        # set inputs to properties
+        self.path_to_input = path_to_input
+        self.path_to_s1 = path_to_s1
+        self.len_of_samples = len_of_samples
+        self.fs = fs
+        self.train_flag=train_flag
+        # count the number of samples in your data set (depending on your disk,
+        #                                               this can take some time)
+        self.count_samples()
+        # create iterable tf.data.Dataset object
+        self.create_tf_data_obj()
+        
+    def count_samples(self):
+        '''
+        Method to list the data of the dataset and count the number of samples. 
+        '''
+
+        # list .wav files in directory
+        self.file_names = fnmatch.filter(os.listdir(self.path_to_input), '*.wav')
+        # count the number of samples contained in the dataset
+        self.total_samples = 0
+        for file in self.file_names:
+            info = WavInfoReader(os.path.join(self.path_to_input, file))
+            self.total_samples = self.total_samples + \
+                int(np.fix(info.data.frame_count/self.len_of_samples))
+    
+         
+    def create_generator(self):
+        '''
+        Method to create the iterator. 
+        '''
+
+        # check if training or validation
+        if self.train_flag:
+            shuffle(self.file_names)
+        # iterate over the files  
+        for file in self.file_names:
+            # read the audio files
+            noisy, fs_1 = sf.read(os.path.join(self.path_to_input, file))
+            speech, fs_2 = sf.read(os.path.join(self.path_to_s1, file))
+            # check if the sampling rates are matching the specifications
+            if fs_1 != self.fs or fs_2 != self.fs:
+                raise ValueError('Sampling rates do not match.')
+            if noisy.ndim != 1 or speech.ndim != 1:
+                raise ValueError('Too many audio channels. The DTLN audio_generator \
+                                 only supports single channel audio data.')
+            # count the number of samples in one file
+            num_samples = int(np.fix(noisy.shape[0]/self.len_of_samples))
+            # iterate over the number of samples
+            for idx in range(num_samples):
+                # cut the audio files in chunks
+                in_dat = noisy[int(idx*self.len_of_samples):int((idx+1)*
+                                                        self.len_of_samples)]
+                tar_dat = speech[int(idx*self.len_of_samples):int((idx+1)*
+                                                        self.len_of_samples)]
+                # yield the chunks as float32 data
+                yield in_dat.astype('float32'), tar_dat.astype('float32')
+              
+
+    def create_tf_data_obj(self):
+        '''
+        Method to to create the tf.data.Dataset. 
+        '''
+
+        # creating the tf.data.Dataset from the iterator
+        self.tf_data_set = tf.data.Dataset.from_generator(
+                        self.create_generator,
+                        (tf.float32, tf.float32),
+                        output_shapes=(tf.TensorShape([self.len_of_samples]), \
+                                       tf.TensorShape([self.len_of_samples])),
+                        args=None
+                        )
+
+        
+                
+
+
+class DTLN_model():
+    '''
+    Class to create and train the DTLN model
+    '''
+    
+    def __init__(self):
+        '''
+        Constructor
+        '''
+
+        # defining default cost function
+        self.cost_function = self.snr_cost
+        # empty property for the model
+        self.model = []
+        # defining default parameters
+        self.fs = 16000
+        self.batchsize = 32
+        self.len_samples = 15
+        self.activation = 'sigmoid'
+        self.numUnits = 128
+        self.numLayer = 2
+        self.blockLen = 512
+        self.block_shift = 128
+        self.dropout = 0.25
+        self.lr = 1e-3
+        self.max_epochs = 200
+        self.encoder_size = 256
+        self.eps = 1e-7
+        # reset all seeds to 42 to reduce invariance between training runs
+        os.environ['PYTHONHASHSEED']=str(42)
+        seed(42)
+        np.random.seed(42)
+        tf.random.set_seed(42)
+        # some line to correctly find some libraries in TF 2.x
+        physical_devices = tf.config.experimental.list_physical_devices('GPU')
+        if len(physical_devices) > 0:
+            for device in physical_devices:
+                tf.config.experimental.set_memory_growth(device, enable=True)
+        
+
+    @staticmethod
+    def snr_cost(s_estimate, s_true):
+        '''
+        Static Method defining the cost function. 
+        The negative signal to noise ratio is calculated here. The loss is 
+        always calculated over the last dimension. 
+        '''
+
+        # calculating the SNR
+        snr = tf.reduce_mean(tf.math.square(s_true), axis=-1, keepdims=True) / \
+            (tf.reduce_mean(tf.math.square(s_true-s_estimate), axis=-1, keepdims=True)+1e-7)
+        # using some more lines, because TF has no log10
+        num = tf.math.log(snr) 
+        denom = tf.math.log(tf.constant(10, dtype=num.dtype))
+        loss = -10*(num / (denom))
+        # returning the loss
+        return loss
+        
+
+    def lossWrapper(self):
+        '''
+        A wrapper function which returns the loss function. This is done to
+        to enable additional arguments to the loss function if necessary.
+        '''
+        def lossFunction(y_true,y_pred):
+            # calculating loss and squeezing single dimensions away
+            loss = tf.squeeze(self.cost_function(y_pred,y_true))
+            # calculate mean over batches
+            loss = tf.reduce_mean(loss)
+            # return the loss
+            return loss
+        # returning the loss function as handle
+        return lossFunction
+    
+    
+
+    '''
+    In the following some helper layers are defined.
+    '''  
+    
+    def stftLayer(self, x):
+        '''
+        Method for an STFT helper layer used with a Lambda layer. The layer
+        calculates the STFT on the last dimension and returns the magnitude and
+        phase of the STFT.
+        '''
+        
+        # creating frames from the continuous waveform
+        frames = tf.signal.frame(x, self.blockLen, self.block_shift)
+        # calculating the fft over the time frames. rfft returns NFFT/2+1 bins.
+        stft_dat = tf.signal.rfft(frames)
+        # calculating magnitude and phase from the complex signal
+        mag = tf.abs(stft_dat)
+        phase = tf.math.angle(stft_dat)
+        # returning magnitude and phase as list
+        return [mag, phase]
+    
+    def fftLayer(self, x):
+        '''
+        Method for an fft helper layer used with a Lambda layer. The layer
+        calculates the rFFT on the last dimension and returns the magnitude and
+        phase of the STFT.
+        '''
+        
+        # expanding dimensions
+        frame = tf.expand_dims(x, axis=1)
+        # calculating the fft over the time frames. rfft returns NFFT/2+1 bins.
+        stft_dat = tf.signal.rfft(frame)
+        # calculating magnitude and phase from the complex signal
+        mag = tf.abs(stft_dat)
+        phase = tf.math.angle(stft_dat)
+        # returning magnitude and phase as list
+        return [mag, phase]
+
+ 
+        
+    def ifftLayer(self, x):
+        '''
+        Method for an inverse FFT layer used with an Lambda layer. This layer
+        calculates time domain frames from magnitude and phase information. 
+        As input x a list with [mag,phase] is required.
+        '''
+        
+        # calculating the complex representation
+        s1_stft = (tf.cast(x[0], tf.complex64) * 
+                    tf.exp( (1j * tf.cast(x[1], tf.complex64))))
+        # returning the time domain frames
+        return tf.signal.irfft(s1_stft)  
+    
+    
+    def overlapAddLayer(self, x):
+        '''
+        Method for an overlap and add helper layer used with a Lambda layer.
+        This layer reconstructs the waveform from a framed signal.
+        '''
+
+        # calculating and returning the reconstructed waveform
+        return tf.signal.overlap_and_add(x, self.block_shift)
+    
+        
+
+    def seperation_kernel(self, num_layer, mask_size, x, stateful=False):
+        '''
+        Method to create a separation kernel. 
+        !! Important !!: Do not use this layer with a Lambda layer. If used with
+        a Lambda layer the gradients are updated correctly.
+
+        Inputs:
+            num_layer       Number of LSTM layers
+            mask_size       Output size of the mask and size of the Dense layer
+        '''
+
+        # creating num_layer number of LSTM layers
+        for idx in range(num_layer):
+            x = LSTM(self.numUnits, return_sequences=True, stateful=stateful)(x)
+            # using dropout between the LSTM layer for regularization 
+            if idx<(num_layer-1):
+                x = Dropout(self.dropout)(x)
+        # creating the mask with a Dense and an Activation layer
+        mask = Dense(mask_size)(x)
+        mask = Activation(self.activation)(mask)
+        # returning the mask
+        return mask
+    
+    def seperation_kernel_with_states(self, num_layer, mask_size, x, 
+                                      in_states):
+        '''
+        Method to create a separation kernel, which returns the LSTM states. 
+        !! Important !!: Do not use this layer with a Lambda layer. If used with
+        a Lambda layer the gradients are updated correctly.
+
+        Inputs:
+            num_layer       Number of LSTM layers
+            mask_size       Output size of the mask and size of the Dense layer
+        '''
+        
+        states_h = []
+        states_c = []
+        # creating num_layer number of LSTM layers
+        for idx in range(num_layer):
+            in_state = [in_states[:,idx,:, 0], in_states[:,idx,:, 1]]
+            x, h_state, c_state = LSTM(self.numUnits, return_sequences=True, 
+                     unroll=True, return_state=True)(x, initial_state=in_state)
+            # using dropout between the LSTM layer for regularization 
+            if idx<(num_layer-1):
+                x = Dropout(self.dropout)(x)
+            states_h.append(h_state)
+            states_c.append(c_state)
+        # creating the mask with a Dense and an Activation layer
+        mask = Dense(mask_size)(x)
+        mask = Activation(self.activation)(mask)
+        out_states_h = tf.reshape(tf.stack(states_h, axis=0), 
+                                  [1,num_layer,self.numUnits])
+        out_states_c = tf.reshape(tf.stack(states_c, axis=0), 
+                                  [1,num_layer,self.numUnits])
+        out_states = tf.stack([out_states_h, out_states_c], axis=-1)
+        # returning the mask and states
+        return mask, out_states
+
+    def build_DTLN_model(self, norm_stft=False):
+        '''
+        Method to build and compile the DTLN model. The model takes time domain 
+        batches of size (batchsize, len_in_samples) and returns enhanced clips 
+        in the same dimensions. As optimizer for the Training process the Adam
+        optimizer with a gradient norm clipping of 3 is used. 
+        The model contains two separation cores. The first has an STFT signal 
+        transformation and the second a learned transformation based on 1D-Conv 
+        layer. 
+        '''
+        
+        # input layer for time signal
+        time_dat = Input(batch_shape=(None, None))
+        # calculate STFT
+        mag,angle = Lambda(self.stftLayer)(time_dat)
+        # normalizing log magnitude stfts to get more robust against level variations
+        if norm_stft:
+            mag_norm = InstantLayerNormalization()(tf.math.log(mag + 1e-7))
+        else:
+            # behaviour like in the paper
+            mag_norm = mag
+        # predicting mask with separation kernel  
+        mask_1 = self.seperation_kernel(self.numLayer, (self.blockLen//2+1), mag_norm)
+        # multiply mask with magnitude
+        estimated_mag = Multiply()([mag, mask_1])
+        # transform frames back to time domain
+        estimated_frames_1 = Lambda(self.ifftLayer)([estimated_mag,angle])
+        # encode time domain frames to feature domain
+        encoded_frames = Conv1D(self.encoder_size,1,strides=1,use_bias=False)(estimated_frames_1)
+        # normalize the input to the separation kernel
+        encoded_frames_norm = InstantLayerNormalization()(encoded_frames)
+        # predict mask based on the normalized feature frames
+        mask_2 = self.seperation_kernel(self.numLayer, self.encoder_size, encoded_frames_norm)
+        # multiply encoded frames with the mask
+        estimated = Multiply()([encoded_frames, mask_2]) 
+        # decode the frames back to time domain
+        decoded_frames = Conv1D(self.blockLen, 1, padding='causal',use_bias=False)(estimated)
+        # create waveform with overlap and add procedure
+        estimated_sig = Lambda(self.overlapAddLayer)(decoded_frames)
+
+        
+        # create the model
+        self.model = Model(inputs=time_dat, outputs=estimated_sig)
+        # show the model summary
+        print(self.model.summary())
+        
+    def build_DTLN_model_stateful(self, norm_stft=False):
+        '''
+        Method to build stateful DTLN model for real time processing. The model 
+        takes one time domain frame of size (1, blockLen) and one enhanced frame. 
+         
+        '''
+        
+        # input layer for time signal
+        time_dat = Input(batch_shape=(1, self.blockLen))
+        # calculate STFT
+        mag,angle = Lambda(self.fftLayer)(time_dat)
+        # normalizing log magnitude stfts to get more robust against level variations
+        if norm_stft:
+            mag_norm = InstantLayerNormalization()(tf.math.log(mag + 1e-7))
+        else:
+            # behaviour like in the paper
+            mag_norm = mag
+        # predicting mask with separation kernel  
+        mask_1 = self.seperation_kernel(self.numLayer, (self.blockLen//2+1), mag_norm, stateful=True)
+        # multiply mask with magnitude
+        estimated_mag = Multiply()([mag, mask_1])
+        # transform frames back to time domain
+        estimated_frames_1 = Lambda(self.ifftLayer)([estimated_mag,angle])
+        # encode time domain frames to feature domain
+        encoded_frames = Conv1D(self.encoder_size,1,strides=1,use_bias=False)(estimated_frames_1)
+        # normalize the input to the separation kernel
+        encoded_frames_norm = InstantLayerNormalization()(encoded_frames)
+        # predict mask based on the normalized feature frames
+        mask_2 = self.seperation_kernel(self.numLayer, self.encoder_size, encoded_frames_norm, stateful=True)
+        # multiply encoded frames with the mask
+        estimated = Multiply()([encoded_frames, mask_2]) 
+        # decode the frames back to time domain
+        decoded_frame = Conv1D(self.blockLen, 1, padding='causal',use_bias=False)(estimated)
+        # create the model
+        self.model = Model(inputs=time_dat, outputs=decoded_frame)
+        # show the model summary
+        print(self.model.summary())
+        
+    def compile_model(self):
+        '''
+        Method to compile the model for training
+
+        '''
+        
+        # use the Adam optimizer with a clipnorm of 3
+        optimizerAdam = keras.optimizers.Adam(lr=self.lr, clipnorm=3.0)
+        # compile model with loss function
+        self.model.compile(loss=self.lossWrapper(), optimizer=optimizerAdam)
+        
+    def create_saved_model(self, weights_file, target_name):
+        '''
+        Method to create a saved model folder from a weights file
+
+        '''
+        # check for type
+        if weights_file.find('_norm_') != -1:
+            norm_stft = True
+        else:
+            norm_stft = False
+        # build model    
+        self.build_DTLN_model_stateful(norm_stft=norm_stft)
+        # load weights
+        self.model.load_weights(weights_file)
+        # save model
+        tf.saved_model.save(self.model, target_name)
+        
+    def create_tf_lite_model(self, weights_file, target_name, use_dynamic_range_quant=False):
+        '''
+        Method to create a tf lite model folder from a weights file. 
+        The conversion creates two models, one for each separation core. 
+        Tf lite does not support complex numbers yet. Some processing must be 
+        done outside the model.
+        For further information and how real time processing can be 
+        implemented see "real_time_processing_tf_lite.py".
+        
+        The conversion only works with TF 2.3.
+
+        '''
+        # check for type
+        if weights_file.find('_norm_') != -1:
+            norm_stft = True
+            num_elements_first_core = 2 + self.numLayer * 3 + 2
+        else:
+            norm_stft = False
+            num_elements_first_core = self.numLayer * 3 + 2
+        # build model    
+        self.build_DTLN_model_stateful(norm_stft=norm_stft)
+        # load weights
+        self.model.load_weights(weights_file)
+        
+        #### Model 1 ##########################
+        mag = Input(batch_shape=(1, 1, (self.blockLen//2+1)))
+        states_in_1 = Input(batch_shape=(1, self.numLayer, self.numUnits, 2))
+        # normalizing log magnitude stfts to get more robust against level variations
+        if norm_stft:
+            mag_norm = InstantLayerNormalization()(tf.math.log(mag + 1e-7))
+        else:
+            # behaviour like in the paper
+            mag_norm = mag
+        # predicting mask with separation kernel  
+        mask_1, states_out_1 = self.seperation_kernel_with_states(self.numLayer, 
+                                                    (self.blockLen//2+1), 
+                                                    mag_norm, states_in_1)
+        
+        model_1 = Model(inputs=[mag, states_in_1], outputs=[mask_1, states_out_1])
+        
+        #### Model 2 ###########################
+        
+        estimated_frame_1 = Input(batch_shape=(1, 1, (self.blockLen)))
+        states_in_2 = Input(batch_shape=(1, self.numLayer, self.numUnits, 2))
+        
+        # encode time domain frames to feature domain
+        encoded_frames = Conv1D(self.encoder_size,1,strides=1,
+                                use_bias=False)(estimated_frame_1)
+        # normalize the input to the separation kernel
+        encoded_frames_norm = InstantLayerNormalization()(encoded_frames)
+        # predict mask based on the normalized feature frames
+        mask_2, states_out_2 = self.seperation_kernel_with_states(self.numLayer, 
+                                                    self.encoder_size, 
+                                                    encoded_frames_norm, 
+                                                    states_in_2)
+        # multiply encoded frames with the mask
+        estimated = Multiply()([encoded_frames, mask_2]) 
+        # decode the frames back to time domain
+        decoded_frame = Conv1D(self.blockLen, 1, padding='causal',
+                               use_bias=False)(estimated)
+        
+        model_2 = Model(inputs=[estimated_frame_1, states_in_2], 
+                        outputs=[decoded_frame, states_out_2])
+        
+        # set weights to submodels
+        weights = self.model.get_weights()
+        model_1.set_weights(weights[:num_elements_first_core])
+        model_2.set_weights(weights[num_elements_first_core:])
+        # convert first model
+        converter = tf.lite.TFLiteConverter.from_keras_model(model_1)
+        if use_dynamic_range_quant:
+            converter.optimizations = [tf.lite.Optimize.DEFAULT]
+        tflite_model = converter.convert()
+        with tf.io.gfile.GFile(target_name + '_1.tflite', 'wb') as f:
+              f.write(tflite_model)
+        # convert second model    
+        converter = tf.lite.TFLiteConverter.from_keras_model(model_2)
+        if use_dynamic_range_quant:
+            converter.optimizations = [tf.lite.Optimize.DEFAULT]
+        tflite_model = converter.convert()
+        with tf.io.gfile.GFile(target_name + '_2.tflite', 'wb') as f:
+              f.write(tflite_model)
+              
+        print('TF lite conversion complete!')
+        
+    
+    def train_model(self, runName, path_to_train_mix, path_to_train_speech, \
+                    path_to_val_mix, path_to_val_speech):
+        '''
+        Method to train the DTLN model. 
+        '''
+        
+        # create save path if not existent
+        savePath = './models_'+ runName+'/' 
+        if not os.path.exists(savePath):
+            os.makedirs(savePath)
+        # create log file writer
+        csv_logger = CSVLogger(savePath+ 'training_' +runName+ '.log')
+        # create callback for the adaptive learning rate
+        reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5,
+                              patience=3, min_lr=10**(-10), cooldown=1)
+        # create callback for early stopping
+        early_stopping = EarlyStopping(monitor='val_loss', min_delta=0, 
+            patience=10, verbose=0, mode='auto', baseline=None)
+        # create model check pointer to save the best model
+        checkpointer = ModelCheckpoint(savePath+runName+'.h5',
+                                       monitor='val_loss',
+                                       verbose=1,
+                                       save_best_only=True,
+                                       save_weights_only=True,
+                                       mode='auto',
+                                       save_freq='epoch'
+                                       )
+
+        # calculate length of audio chunks in samples
+        len_in_samples = int(np.fix(self.fs * self.len_samples / 
+                                    self.block_shift)*self.block_shift)
+        # create data generator for training data
+        generator_input = audio_generator(path_to_train_mix, 
+                                          path_to_train_speech, 
+                                          len_in_samples, 
+                                          self.fs, train_flag=True)
+        dataset = generator_input.tf_data_set
+        dataset = dataset.batch(self.batchsize, drop_remainder=True).repeat()
+        # calculate number of training steps in one epoch
+        steps_train = generator_input.total_samples//self.batchsize
+        # create data generator for validation data
+        generator_val = audio_generator(path_to_val_mix,
+                                        path_to_val_speech, 
+                                        len_in_samples, self.fs)
+        dataset_val = generator_val.tf_data_set
+        dataset_val = dataset_val.batch(self.batchsize, drop_remainder=True).repeat()
+        # calculate number of validation steps
+        steps_val = generator_val.total_samples//self.batchsize
+        # start the training of the model
+        self.model.fit(
+            x=dataset, 
+            batch_size=None,
+            steps_per_epoch=steps_train, 
+            epochs=self.max_epochs,
+            verbose=1,
+            validation_data=dataset_val,
+            validation_steps=steps_val, 
+            callbacks=[checkpointer, reduce_lr, csv_logger, early_stopping],
+            max_queue_size=50,
+            workers=4,
+            use_multiprocessing=True)
+        # clear out garbage
+        tf.keras.backend.clear_session()
+
+    
+
+class InstantLayerNormalization(Layer):
+    '''
+    Class implementing instant layer normalization. It can also be called 
+    channel-wise layer normalization and was proposed by 
+    Luo & Mesgarani (https://arxiv.org/abs/1809.07454v2) 
+    '''
+
+    def __init__(self, **kwargs):
+        '''
+            Constructor
+        '''
+        super(InstantLayerNormalization, self).__init__(**kwargs)
+        self.epsilon = 1e-7 
+        self.gamma = None
+        self.beta = None
+
+    def build(self, input_shape):
+        '''
+        Method to build the weights.
+        '''
+        shape = input_shape[-1:]
+        # initialize gamma
+        self.gamma = self.add_weight(shape=shape,
+                             initializer='ones',
+                             trainable=True,
+                             name='gamma')
+        # initialize beta
+        self.beta = self.add_weight(shape=shape,
+                             initializer='zeros',
+                             trainable=True,
+                             name='beta')
+ 
+
+    def call(self, inputs):
+        '''
+        Method to call the Layer. All processing is done here.
+        '''
+
+        # calculate mean of each frame
+        mean = tf.math.reduce_mean(inputs, axis=[-1], keepdims=True)
+        # calculate variance of each frame
+        variance = tf.math.reduce_mean(tf.math.square(inputs - mean), 
+                                       axis=[-1], keepdims=True)
+        # calculate standard deviation
+        std = tf.math.sqrt(variance + self.epsilon)
+        # normalize each frame independently 
+        outputs = (inputs - mean) / std
+        # scale with gamma
+        outputs = outputs * self.gamma
+        # add the bias beta
+        outputs = outputs + self.beta
+        # return output
+        return outputs
+    
+
+    

models/DTLN (yash-04)/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Nils L. Westhausen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

models/DTLN (yash-04)/README.md

@@ -0,0 +1,302 @@
+# Dual-signal Transformation LSTM Network
+
++ Tensorflow 2.x implementation of the stacked dual-signal transformation LSTM network (DTLN) for real-time noise suppression.
++ This repository provides the code for training, infering and serving the DTLN model in python. It also provides pretrained models in SavedModel, TF-lite and ONNX format, which can be used as baseline for your own projects. The model is able to run with real time audio on a RaspberryPi.
++ If you are doing cool things with this repo, tell me about it. I am always curious about what you are doing with this code or this models.
+
+---
+
+The DTLN model was handed in to the deep noise suppression challenge ([DNS-Challenge](https://github.com/microsoft/DNS-Challenge)) and the paper was presented at Interspeech 2020. 
+
+
+This approach combines a short-time Fourier transform (STFT) and a learned analysis and synthesis basis in a stacked-network approach with less than one million parameters. The model was trained on 500h of noisy speech provided by the challenge organizers. The network is capable of real-time processing (one frame in, one frame out) and reaches competitive results.
+Combining these two types of signal transformations enables the DTLN to robustly extract information from magnitude spectra and incorporate phase information from the learned feature basis. The method shows state-of-the-art performance and outperforms the DNS-Challenge baseline by 0.24 points absolute in terms of the mean opinion score (MOS).
+
+For more information see the [paper](https://www.isca-speech.org/archive/interspeech_2020/westhausen20_interspeech.html). The results of the DNS-Challenge are published [here](https://www.microsoft.com/en-us/research/academic-program/deep-noise-suppression-challenge-interspeech-2020/#!results). We reached a competitive 8th place out of 17 teams in the real time track.
+
+---
+
+For baseline usage and to reproduce the processing used for the paper run:
+```bash
+$ python run_evaluation.py -i in/folder/with/wav -o target/folder/processed/files -m ./pretrained_model/model.h5
+```
+
+---
+
+The pretrained DTLN-aec (the DTLN applied to acoustic echo cancellation) can be found in the [DTLN-aec repository](https://github.com/breizhn/DTLN-aec).
+
+---
+
+Author: Nils L. Westhausen ([Communication Acoustics](https://uol.de/en/kommunikationsakustik) , Carl von Ossietzky University, Oldenburg, Germany)
+
+This code is licensed under the terms of the MIT license.
+
+
+---
+### Citing:
+
+If you are using the DTLN model, please cite:
+
+```BibTex
+@inproceedings{Westhausen2020,
+  author={Nils L. Westhausen and Bernd T. Meyer},
+  title={{Dual-Signal Transformation LSTM Network for Real-Time Noise Suppression}},
+  year=2020,
+  booktitle={Proc. Interspeech 2020},
+  pages={2477--2481},
+  doi={10.21437/Interspeech.2020-2631},
+  url={http://dx.doi.org/10.21437/Interspeech.2020-2631}
+}
+```
+
+
+---
+### Contents of the README:
+
+* [Results](#results)
+* [Execution Times](#execution-times)
+* [Audio Samples](#audio-samples)
+* [Contents of the repository](#contents-of-the-repository)
+* [Python dependencies](#python-dependencies)
+* [Training data preparation](#training-data-preparation)
+* [Run a training of the DTLN model](#run-a-training-of-the-dtln-model)
+* [Measuring the execution time of the DTLN model with the SavedModel format](#measuring-the-execution-time-of-the-dtln-model-with-the-savedmodel-format)
+* [Real time processing with the SavedModel format](#real-time-processing-with-the-savedmodel-format)
+* [Real time processing with tf-lite](#real-time-processing-with-tf-lite)
+* [Real time audio with sounddevice and tf-lite](#real-time-audio-with-sounddevice-and-tf-lite)
+* [Model conversion and real time processing with ONNX](#model-conversion-and-real-time-processing-with-onnx)
+
+
+
+---
+### Results:
+
+Results on the DNS-Challenge non reverberant test set:
+Model | PESQ [mos] | STOI [%] | SI-SDR [dB] | TF version
+--- | --- | --- | --- | ---
+unprocessed | 2.45 | 91.52 | 9.07 |
+NsNet (Baseline) | 2.70 | 90.56 | 12.57 |
+ |  |  |  | 
+DTLN (500h) | 3.04 | 94.76 | 16.34 | 2.1
+DTLN (500h)| 2.98 | 94.75 | 16.20 | TF-light
+DTLN (500h) | 2.95 | 94.47 | 15.71 | TF-light quantized
+ |  |  |  | 
+DTLN norm (500h) | 3.04 | 94.47 | 16.10 | 2.2
+ |  |  |  | 
+DTLN norm (40h) | 3.05 | 94.57 | 16.88 | 2.2
+DTLN norm (40h) | 2.98 | 94.56 | 16.58 | TF-light
+DTLN norm (40h) | 2.98 | 94.51 | 16.22 | TF-light quantized
+
+* The conversion to TF-light slightly reduces the performance. 
+* The dynamic range quantization of TF-light also reduces the performance a bit and introduces some quantization noise. But the audio-quality is still on a high level and the model is real-time capable on the Raspberry Pi 3 B+.
+* The normalization of the log magnitude of the STFT does not decrease the model performance and makes it more robust against level variations.
+* With data augmentation during training it is possible to train the DTLN model on just 40h of noise and speech data. If you have any question regarding this, just contact me.
+
+[To contents](#contents-of-the-readme)
+
+---
+
+### Execution Times:
+
+Execution times for SavedModel are measured with TF 2.2 and for TF-lite with the TF-lite runtime:
+System | Processor | #Cores | SavedModel | TF-lite | TF-lite quantized
+--- | --- | --- | --- | --- | ---
+Ubuntu 18.04         | Intel I5 6600k @ 3.5 GHz | 4 | 0.65 ms | 0.36 ms | 0.27 ms
+Macbook Air mid 2012 | Intel I7 3667U @ 2.0 GHz | 2 | 1.4 ms | 0.6 ms | 0.4 ms
+Raspberry Pi 3 B+    | ARM Cortex A53 @ 1.4 GHz | 4 | 15.54 ms | 9.6 ms | 2.2 ms
+
+For real-time capability the execution time must be below 8 ms.
+
+[To contents](#contents-of-the-readme)
+
+---
+
+### Audio Samples:
+
+Here some audio samples created with the tf-lite model. Sadly audio can not be integrated directly into markdown.
+
+Noisy | Enhanced | Noise type
+--- | --- | --- 
+[Sample 1](https://cloudsync.uol.de/s/GFHzmWWJAwgQPLf) | [Sample 1](https://cloudsync.uol.de/s/p3M48y7cjkJ2ZZg) | Air conditioning
+[Sample 2](https://cloudsync.uol.de/s/4Y2PoSpJf7nXx9T) | [Sample 2](https://cloudsync.uol.de/s/QeK4aH5KCELPnko) | Music
+[Sample 3](https://cloudsync.uol.de/s/Awc6oBtnTpb5pY7) | [Sample 3](https://cloudsync.uol.de/s/yNsmDgxH3MPWMTi) | Bus 
+
+
+[To contents](#contents-of-the-readme)
+
+---
+### Contents of the repository:
+
+*  **DTLN_model.py** \
+  This file is containing the model, data generator and the training routine.
+*  **run_training.py** \
+  Script to run the training. Before you can start the training with `$ python run_training.py`you have to set the paths to you training and validation data inside the script. The training script uses a default setup.
+* **run_evaluation.py** \
+  Script to process a folder with optional subfolders containing .wav files with a trained DTLN model. With the pretrained model delivered with this repository a folder can be processed as following: \
+  `$ python run_evaluation.py -i /path/to/input -o /path/for/processed -m ./pretrained_model/model.h5` \
+  The evaluation script will create the new folder with the same structure as the input folder and the files will have the same name as the input files.
+* **measure_execution_time.py** \
+  Script for measuring the execution time with the saved DTLN model in `./pretrained_model/dtln_saved_model/`. For further information see this [section](#measuring-the-execution-time-of-the-dtln-model-with-the-savedmodel-format).
+* **real_time_processing.py** \
+  Script, which explains how real time processing with the SavedModel works. For more information see this [section](#real-time-processing-with-the-savedmodel-format).
++  **./pretrained_model/** \
+   * `model.h5`: Model weights as used in the DNS-Challenge DTLN model.
+   * `DTLN_norm_500h.h5`: Model weights trained on 500h with normalization of stft log magnitudes.
+   * `DTLN_norm_40h.h5`: Model weights trained on 40h with normalization of stft log magnitudes.
+   * `./dtln_saved_model`: same as `model.h5` but as a stateful model in SavedModel format.
+   * `./DTLN_norm_500h_saved_model`: same as `DTLN_norm_500h.h5` but as a stateful model in SavedModel format.
+   * `./DTLN_norm_40h_saved_model`: same as `DTLN_norm_40h.h5` but as a stateful model in SavedModel format.
+   * `model_1.tflite` together with `model_2.tflite`: same as `model.h5` but as TF-lite model with external state handling.
+   * `model_quant_1.tflite` together with `model_quant_2.tflite`: same as `model.h5` but as TF-lite model with external state handling and dynamic range quantization.
+   * `model_1.onnx` together with `model_2.onnx`: same as `model.h5` but as ONNX model with external state handling.
+   
+[To contents](#contents-of-the-readme)
+   
+---
+### Python dependencies:
+
+The following packages will be required for this repository:
+* TensorFlow (2.x)
+* librosa
+* wavinfo 
+
+
+All additional packages (numpy, soundfile, etc.) should be installed on the fly when using conda or pip. I recommend using conda environments or [pyenv](https://github.com/pyenv/pyenv) [virtualenv](https://github.com/pyenv/pyenv-virtualenv) for the python environment. For training a GPU with at least 5 GB of memory is required. I recommend at least Tensorflow 2.1 with Nvidia driver 418 and Cuda 10.1. If you use conda Cuda will be installed on the fly and you just need the driver. For evaluation-only the CPU version of Tensorflow is enough. Everything was tested on Ubuntu 18.04.
+
+Conda environments for training (with cuda) and for evaluation (CPU only) can be created as following:
+
+For the training environment:
+```shell
+$ conda env create -f train_env.yml
+```
+For the evaluation environment:
+```
+$ conda env create -f eval_env.yml
+```
+For the tf-lite environment:
+```
+$ conda env create -f tflite_env.yml
+```
+The tf-lite runtime must be downloaded from [here](https://www.tensorflow.org/lite/guide/python).
+
+[To contents](#contents-of-the-readme)
+
+---
+### Training data preparation:
+
+1. Clone the forked DNS-Challenge [repository](https://github.com/breizhn/DNS-Challenge). Before cloning the repository make sure `git-lfs` is installed. Also make sure your disk has enough space. I recommend downloading the data to an SSD for faster dataset creation.
+
+2. Run `noisyspeech_synthesizer_multiprocessing.py` to create the dataset. `noisyspeech_synthesizer.cfg`was changed according to my training setup used for the DNS-Challenge. 
+
+3. Run `split_dns_corpus.py`to divide the dataset in training and validation data. The classic 80:20 split is applied. This file was added to the forked repository by me.
+
+[To contents](#contents-of-the-readme)
+
+---
+### Run a training of the DTLN model:
+
+1. Make sure all dependencies are installed in your python environment.
+
+2. Change the paths to your training and validation dataset in `run_training.py`.
+
+3. Run `$ python run_training.py`. 
+
+One epoch takes around 21 minutes on a Nvidia RTX 2080 Ti when loading the training data from an SSD. 
+
+[To contents](#contents-of-the-readme)
+
+---
+### Measuring the execution time of the DTLN model with the SavedModel format:
+
+In total there are three ways to measure the execution time for one block of the model: Running a sequence in Keras and dividing by the number of blocks in the sequence, building a stateful model in Keras and running block by block, and saving the stateful model in Tensorflow's SavedModel format and calling that one block by block. In the following I will explain how running the model in the SavedModel format, because it is the most portable version and can also be called from Tensorflow Serving.
+
+A Keras model can be saved to the saved model format:
+```python
+import tensorflow as tf
+'''
+Building some model here
+'''
+tf.saved_model.save(your_keras_model, 'name_save_path')
+```
+Important here for real time block by block processing is, to make the LSTM layer stateful, so they can remember the states from the previous block.
+
+The model can be imported with 
+```python
+model = tf.saved_model.load('name_save_path')
+```
+
+For inference we now first call this for mapping signature names to functions
+```python
+infer = model.signatures['serving_default']
+```
+
+and now for inferring the block `x` call
+```python
+y = infer(tf.constant(x))['conv1d_1']
+```
+This command gives you the result on the node `'conv1d_1'`which is our output node for real time processing. For more information on using the SavedModel format and obtaining the output node see this [Guide](https://www.tensorflow.org/guide/saved_model).
+
+For making everything easier this repository provides a stateful DTLN SavedModel. 
+For measuring the execution time call:
+```
+$ python measure_execution_time.py
+```
+
+[To contents](#contents-of-the-readme)
+
+---
+
+### Real time processing with the SavedModel format:
+
+For explanation look at `real_time_processing.py`. 
+
+Here some consideration for integrating this model in your project:
+* The sampling rate of this model is fixed at 16 kHz. It will not work smoothly with other sampling rates.
+* The block length of 32 ms and the block shift of 8 ms are also fixed. For changing these values, the model must be retrained.
+* The delay created by the model is the block length, so the input-output delay is 32 ms.
+* For real time capability on your system, the execution time must be below the length of the block shift, so below 8 ms. 
+* If can not give you support on the hardware side, regarding soundcards, drivers and so on. Be aware, a lot of artifacts can come from this side.
+
+[To contents](#contents-of-the-readme)
+
+---
+### Real time processing with tf-lite:
+
+With TF 2.3 it is finally possible to convert LSTMs to tf-lite. It is still not perfect because the states must be handled seperatly for a stateful model and tf-light does not support complex numbers. That means that the model is splitted in two submodels when converting it to tf-lite and the calculation of the FFT and iFFT is performed outside the model. I provided an example script for explaining, how real time processing with the tf light model works (```real_time_processing_tf_lite.py```). In this script the tf-lite runtime is used. The runtime can be downloaded [here](https://www.tensorflow.org/lite/guide/python). Quantization works now.
+
+Using the tf-lite DTLN model and the tf-lite runtime the execution time on an old Macbook Air mid 2012 can be decreased to **0.6 ms**.
+
+[To contents](#contents-of-the-readme)
+
+---
+### Real time audio with sounddevice and tf-lite:
+
+The file ```real_time_dtln_audio.py```is an example how real time audio with the tf-lite model and the [sounddevice](https://github.com/spatialaudio/python-sounddevice) toolbox can be implemented. The script is based on the ```wire.py``` example. It works fine on an old Macbook Air mid 2012 and so it will probably run on most newer devices. In the quantized version it was sucessfully tested on an Raspberry Pi 3B +.
+
+First check for your audio devices:
+```
+$ python real_time_dtln_audio.py --list-devices
+```
+Choose the index of an input and an output device and call:
+```
+$ python real_time_dtln_audio.py -i in_device_idx -o out_device_idx
+```
+If the script is showing too much ```input underflow``` restart the sript. If that does not help, increase the latency with the ```--latency``` option. The default value is 0.2 .
+
+[To contents](#contents-of-the-readme)
+
+---
+### Model conversion and real time processing with ONNX:
+
+Finally I got the ONNX model working. 
+For converting the model TF 2.1 and keras2onnx is required. keras2onnx can be downloaded [here](https://github.com/onnx/keras-onnx) and must be installed from source as described in the README. When all dependencies are installed, call:
+```
+$ python convert_weights_to_onnx.py -m /name/of/the/model.h5 -t onnx_model_name
+```
+to convert the model to the ONNX format. The model is split in two parts as for the TF-lite model. The conversion does not work on MacOS.
+The real time processing works similar to the TF-lite model and can be looked up in following file: ```real_time_processing_onnx.py ```
+The ONNX runtime required for this script can be installed with:
+```
+$ pip install onnxruntime
+```
+The execution time on the Macbook Air mid 2012 is around 1.13 ms for one block.

models/DTLN (yash-04)/convert_weights_to_onnx.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+"""
+Script to covert a .h5 weights file of the DTLN model to ONNX.
+At the moment the conversion only works with TF 2.1 and not on Mac.
+
+Example call:
+    $python convert_weights_to_ONNX.py -m /name/of/the/model.h5 \
+                                              -t name_target 
+                              
+
+Author: Nils L. Westhausen (nils.westhausen@uol.de)
+Version: 03.07.2020
+
+This code is licensed under the terms of the MIT-license.
+"""
+
+from DTLN_model import DTLN_model, InstantLayerNormalization
+import argparse
+from tensorflow.keras.models import Model
+from tensorflow.keras.layers import Input, Multiply, Conv1D
+import tensorflow as tf
+import keras2onnx
+
+
+if __name__ == '__main__':
+    # arguement parser for running directly from the command line
+    parser = argparse.ArgumentParser(description='data evaluation')
+    parser.add_argument('--weights_file', '-m',
+                        help='path to .h5 weights file')
+    parser.add_argument('--target_folder', '-t',
+                        help='target folder for saved model')
+    
+    args = parser.parse_args()
+    weights_file = args.weights_file
+    dtln_class = DTLN_model()
+    # check for type
+    if weights_file.find('_norm_') != -1:
+        norm_stft = True
+        num_elements_first_core = 2 + dtln_class.numLayer * 3 + 2
+    else:
+        norm_stft = False
+        num_elements_first_core = dtln_class.numLayer * 3 + 2
+    # build model    
+    dtln_class.build_DTLN_model_stateful(norm_stft=norm_stft)
+    # load weights
+    dtln_class.model.load_weights(weights_file)
+    #### Model 1 ##########################
+    mag = Input(batch_shape=(1, 1, (dtln_class.blockLen//2+1)))
+    states_in_1 = Input(batch_shape=(1, dtln_class.numLayer, dtln_class.numUnits, 2))
+    # normalizing log magnitude stfts to get more robust against level variations
+    if norm_stft:
+        mag_norm = InstantLayerNormalization()(tf.math.log(mag + 1e-7))
+    else:
+        # behaviour like in the paper
+        mag_norm = mag
+    # predicting mask with separation kernel  
+    mask_1, states_out_1 = dtln_class.seperation_kernel_with_states(dtln_class.numLayer, 
+                                                (dtln_class.blockLen//2+1), 
+                                                mag_norm, states_in_1)
+    
+    model_1 = Model(inputs=[mag, states_in_1], outputs=[mask_1, states_out_1])
+    
+    #### Model 2 ###########################
+    
+    estimated_frame_1 = Input(batch_shape=(1, 1, (dtln_class.blockLen)))
+    states_in_2 = Input(batch_shape=(1, dtln_class.numLayer, dtln_class.numUnits, 2))
+    
+    # encode time domain frames to feature domain
+    encoded_frames = Conv1D(dtln_class.encoder_size,1,strides=1,
+                            use_bias=False)(estimated_frame_1)
+    # normalize the input to the separation kernel
+    encoded_frames_norm = InstantLayerNormalization()(encoded_frames)
+    # predict mask based on the normalized feature frames
+    mask_2, states_out_2 = dtln_class.seperation_kernel_with_states(dtln_class.numLayer, 
+                                                dtln_class.encoder_size, 
+                                                encoded_frames_norm, 
+                                                states_in_2)
+    # multiply encoded frames with the mask
+    estimated = Multiply()([encoded_frames, mask_2]) 
+    # decode the frames back to time domain
+    decoded_frame = Conv1D(dtln_class.blockLen, 1, padding='causal',
+                           use_bias=False)(estimated)
+    
+    model_2 = Model(inputs=[estimated_frame_1, states_in_2], 
+                    outputs=[decoded_frame, states_out_2])
+    
+    # set weights to submodels
+    weights = dtln_class.model.get_weights()
+    
+    model_1.set_weights(weights[:num_elements_first_core])
+    model_2.set_weights(weights[num_elements_first_core:])
+    # convert first model
+    onnx_model = keras2onnx.convert_keras(model_1)
+    temp_model_file = args.target_folder + '_1.onnx'
+    keras2onnx.save_model(onnx_model, temp_model_file)
+    # convert second model
+    onnx_model = keras2onnx.convert_keras(model_2)
+    temp_model_file = args.target_folder + '_2.onnx'
+    keras2onnx.save_model(onnx_model, temp_model_file)
+    
+          
+    print('ONNX conversion complete!')

models/DTLN (yash-04)/convert_weights_to_saved_model.py

@@ -0,0 +1,30 @@
+"""
+Script to covert a .h weights file of the DTLN model to the saved model format.
+
+Example call:
+    $python convert_weights_to_saved_model.py -m /name/of/the/model.h5 \
+                                              -t name_target_folder 
+                              
+
+Author: Nils L. Westhausen (nils.westhausen@uol.de)
+Version: 24.06.2020
+
+This code is licensed under the terms of the MIT-license.
+"""
+
+from DTLN_model import DTLN_model
+import argparse
+
+
+if __name__ == '__main__':
+    # arguement parser for running directly from the command line
+    parser = argparse.ArgumentParser(description='data evaluation')
+    parser.add_argument('--weights_file', '-m',
+                        help='path to .h5 weights file')
+    parser.add_argument('--target_folder', '-t',
+                        help='target folder for saved model')
+    
+    args = parser.parse_args()
+    
+    converter = DTLN_model()
+    converter.create_saved_model(args.weights_file, args.target_folder)

models/DTLN (yash-04)/convert_weights_to_tf_lite.py

@@ -0,0 +1,41 @@
+"""
+Script to covert a .h5 weights file of the DTLN model to tf lite.
+
+Example call:
+    $python convert_weights_to_tf_light.py -m /name/of/the/model.h5 \
+                                              -t name_target 
+                              
+
+Author: Nils L. Westhausen (nils.westhausen@uol.de)
+Version: 30.06.2020
+
+This code is licensed under the terms of the MIT-license.
+"""
+
+from DTLN_model import DTLN_model
+import argparse
+from pkg_resources import parse_version
+import tensorflow as tf
+
+
+if __name__ == '__main__':
+    # arguement parser for running directly from the command line
+    parser = argparse.ArgumentParser(description='data evaluation')
+    parser.add_argument('--weights_file', '-m',
+                        help='path to .h5 weights file')
+    parser.add_argument('--target_folder', '-t',
+                        help='target folder for saved model')
+    parser.add_argument('--quantization', '-q',
+                        help='use quantization (True/False)',
+                        default='False')
+    
+    args = parser.parse_args()
+    if parse_version(tf.__version__) < parse_version('2.3.0-rc0'):
+        raise ValueError('Tf version < 2.3. Conversion of LSTMs will not work'+ 
+                         +' with older tensorflow versions')
+    
+    
+    converter = DTLN_model()
+    converter.create_tf_lite_model(args.weights_file, 
+                                   args.target_folder, 
+                                   use_dynamic_range_quant=bool(args.quantization))

models/DTLN (yash-04)/eval_env.yml

@@ -0,0 +1,117 @@
+name: eval_env
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=main
+  - _tflow_select=2.3.0=mkl
+  - absl-py=0.9.0=py37_0
+  - astunparse=1.6.3=py_0
+  - audioread=2.1.8=py37hc8dfbb8_2
+  - blas=1.0=mkl
+  - blinker=1.4=py37_0
+  - brotlipy=0.7.0=py37h7b6447c_1000
+  - bzip2=1.0.8=h516909a_2
+  - c-ares=1.15.0=h7b6447c_1001
+  - ca-certificates=2020.4.5.2=hecda079_0
+  - cachetools=4.1.0=py_1
+  - certifi=2020.4.5.2=py37hc8dfbb8_0
+  - cffi=1.14.0=py37he30daa8_1
+  - chardet=3.0.4=py37_1003
+  - click=7.1.2=py_0
+  - cryptography=2.9.2=py37h1ba5d50_0
+  - cycler=0.10.0=py_2
+  - decorator=4.4.2=py_0
+  - ffmpeg=4.2.3=h167e202_0
+  - freetype=2.10.2=he06d7ca_0
+  - gast=0.3.3=py_0
+  - gettext=0.19.8.1=h5e8e0c9_1
+  - gmp=6.2.0=he1b5a44_2
+  - gnutls=3.6.13=h79a8f9a_0
+  - google-auth=1.14.1=py_0
+  - google-auth-oauthlib=0.4.1=py_2
+  - google-pasta=0.2.0=py_0
+  - grpcio=1.27.2=py37hf8bcb03_0
+  - h5py=2.10.0=py37h7918eee_0
+  - hdf5=1.10.4=hb1b8bf9_0
+  - icu=58.2=hf484d3e_1000
+  - idna=2.9=py_1
+  - intel-openmp=2020.1=217
+  - joblib=0.15.1=py_0
+  - keras-preprocessing=1.1.0=py_1
+  - kiwisolver=1.2.0=py37h99015e2_0
+  - lame=3.100=h14c3975_1001
+  - ld_impl_linux-64=2.33.1=h53a641e_7
+  - libedit=3.1.20181209=hc058e9b_0
+  - libffi=3.3=he6710b0_1
+  - libflac=1.3.3=he1b5a44_0
+  - libgcc-ng=9.1.0=hdf63c60_0
+  - libgfortran-ng=7.3.0=hdf63c60_0
+  - libiconv=1.15=h516909a_1006
+  - libllvm8=8.0.1=hc9558a2_0
+  - libogg=1.3.2=h516909a_1002
+  - libpng=1.6.37=hed695b0_1
+  - libprotobuf=3.12.3=hd408876_0
+  - librosa=0.7.2=py_1
+  - libsndfile=1.0.28=he1b5a44_1000
+  - libstdcxx-ng=9.1.0=hdf63c60_0
+  - libvorbis=1.3.6=he1b5a44_2
+  - llvmlite=0.31.0=py37h5202443_1
+  - markdown=3.1.1=py37_0
+  - matplotlib-base=3.2.1=py37hef1b27d_0
+  - mkl=2020.1=217
+  - mkl-service=2.3.0=py37he904b0f_0
+  - mkl_fft=1.1.0=py37h23d657b_0
+  - mkl_random=1.1.1=py37h0573a6f_0
+  - ncurses=6.2=he6710b0_1
+  - nettle=3.4.1=h1bed415_1002
+  - numba=0.48.0=py37hb3f55d8_0
+  - numpy=1.18.1=py37h4f9e942_0
+  - numpy-base=1.18.1=py37hde5b4d6_1
+  - oauthlib=3.1.0=py_0
+  - openh264=2.1.1=h8b12597_0
+  - openssl=1.1.1g=h516909a_0
+  - opt_einsum=3.1.0=py_0
+  - pip=20.1.1=py37_1
+  - protobuf=3.12.3=py37he6710b0_0
+  - pyasn1=0.4.8=py_0
+  - pyasn1-modules=0.2.7=py_0
+  - pycparser=2.20=py_0
+  - pyjwt=1.7.1=py37_0
+  - pyopenssl=19.1.0=py37_0
+  - pyparsing=2.4.7=pyh9f0ad1d_0
+  - pysocks=1.7.1=py37_0
+  - pysoundfile=0.10.2=py_1001
+  - python=3.7.7=hcff3b4d_5
+  - python-dateutil=2.8.1=py_0
+  - python_abi=3.7=1_cp37m
+  - readline=8.0=h7b6447c_0
+  - requests=2.23.0=py37_0
+  - requests-oauthlib=1.3.0=py_0
+  - resampy=0.2.2=py_0
+  - rsa=4.0=py_0
+  - scikit-learn=0.22.1=py37hd81dba3_0
+  - scipy=1.4.1=py37h0b6359f_0
+  - setuptools=47.3.0=py37_0
+  - six=1.15.0=py_0
+  - sqlite=3.31.1=h62c20be_1
+  - tensorboard=2.2.1=pyh532a8cf_0
+  - tensorboard-plugin-wit=1.6.0=py_0
+  - tensorflow=2.2.0=mkl_py37h6e9ce2d_0
+  - tensorflow-base=2.2.0=mkl_py37hd506778_0
+  - tensorflow-estimator=2.2.0=pyh208ff02_0
+  - termcolor=1.1.0=py37_1
+  - tk=8.6.8=hbc83047_0
+  - tornado=6.0.4=py37h8f50634_1
+  - urllib3=1.25.9=py_0
+  - werkzeug=1.0.1=py_0
+  - wheel=0.34.2=py37_0
+  - wrapt=1.12.1=py37h7b6447c_1
+  - x264=1!152.20180806=h14c3975_0
+  - xz=5.2.5=h7b6447c_0
+  - zlib=1.2.11=h7b6447c_3
+  - pip:
+    - lxml==4.5.1
+    - wavinfo==1.5
+prefix: /home/nils/anaconda3/envs/eval_env
+

models/DTLN (yash-04)/measure_execution_time.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+This script tests the execution time of the DTLN model on a CPU.
+Please use TF 2.2 for comparability.
+
+Just run "python measure_execution_time.py"
+
+Author: Nils L. Westhausen (nils.westhausen@uol.de)
+Version: 13.05.2020
+
+This code is licensed under the terms of the MIT-license.
+"""
+
+import time
+import tensorflow as tf 
+import numpy as np
+import os
+
+# only use the cpu
+os.environ["CUDA_VISIBLE_DEVICES"]=''
+
+if __name__ == '__main__':
+    # loading model in saved model format
+    model = tf.saved_model.load('./pretrained_model/dtln_saved_model')
+    # mapping signature names to functions
+    infer = model.signatures["serving_default"]
+        
+    exec_time = []
+    # create random input for testing
+    x = np.random.randn(1,512).astype('float32')
+    for idx in range(1010):
+        # run timer
+        start_time = time.time()
+        # infer one block
+        y = infer(tf.constant(x))['conv1d_1']
+        exec_time.append((time.time() - start_time))
+    # ignore the first ten iterations
+    print('Execution time per block: ' + 
+          str( np.round(np.mean(np.stack(exec_time[10:]))*1000, 2)) + ' ms')
+
+# Ubuntu 18.04          I5 6600k        @ 3.5 GHz:  0.65 ms (4 cores)
+# Macbook Air mid 2012 	I7 3667U        @ 2.0 GHz:  1.4 ms  (2 cores)
+# Raspberry Pi 3 B+     ARM Cortex A53  @ 1.4 GHz: 15.54    (4 cores)

models/DTLN (yash-04)/real_time_dtln_audio.py

@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+This is a real time example how to implement DTLN tf light model with 
+sounddevice. The script is based on the "wire.py" example of the sounddevice 
+toolbox. If the command line shows "input underflow", restart the script.
+If there are still a lot of dropouts, increase the latency.
+
+First call:
+    
+    $ python real_time_dtln_audio.py --list-devices
+
+to get your audio devices. In the next step call
+
+    $ python real_time_dtln_audio.py -i in_device_idx -o out_device_idx
+
+For .whl files of the tf light runtime go to: 
+    https://www.tensorflow.org/lite/guide/python
+    
+    
+
+Author: Nils L. Westhausen (nils.westhausen@uol.de)
+Version: 01.07.2020
+
+This code is licensed under the terms of the MIT-license.
+"""
+
+
+import numpy as np
+import sounddevice as sd
+import tflite_runtime.interpreter as tflite
+import argparse
+
+
+def int_or_str(text):
+    """Helper function for argument parsing."""
+    try:
+        return int(text)
+    except ValueError:
+        return text
+
+
+parser = argparse.ArgumentParser(add_help=False)
+parser.add_argument(
+    '-l', '--list-devices', action='store_true',
+    help='show list of audio devices and exit')
+args, remaining = parser.parse_known_args()
+if args.list_devices:
+    print(sd.query_devices())
+    parser.exit(0)
+parser = argparse.ArgumentParser(
+    description=__doc__,
+    formatter_class=argparse.RawDescriptionHelpFormatter,
+    parents=[parser])
+parser.add_argument(
+    '-i', '--input-device', type=int_or_str,
+    help='input device (numeric ID or substring)')
+parser.add_argument(
+    '-o', '--output-device', type=int_or_str,
+    help='output device (numeric ID or substring)')
+
+parser.add_argument('--latency', type=float, help='latency in seconds', default=0.2)
+args = parser.parse_args(remaining)
+
+# set some parameters
+block_len_ms = 32 
+block_shift_ms = 8
+fs_target = 16000
+# create the interpreters
+interpreter_1 = tflite.Interpreter(model_path='./pretrained_model/model_1.tflite')
+interpreter_1.allocate_tensors()
+interpreter_2 = tflite.Interpreter(model_path='./pretrained_model/model_2.tflite')
+interpreter_2.allocate_tensors()
+# Get input and output tensors.
+input_details_1 = interpreter_1.get_input_details()
+output_details_1 = interpreter_1.get_output_details()
+input_details_2 = interpreter_2.get_input_details()
+output_details_2 = interpreter_2.get_output_details()
+# create states for the lstms
+states_1 = np.zeros(input_details_1[1]['shape']).astype('float32')
+states_2 = np.zeros(input_details_2[1]['shape']).astype('float32')
+# calculate shift and length
+block_shift = int(np.round(fs_target * (block_shift_ms / 1000)))
+block_len = int(np.round(fs_target * (block_len_ms / 1000)))
+# create buffer
+in_buffer = np.zeros((block_len)).astype('float32')
+out_buffer = np.zeros((block_len)).astype('float32')
+
+
+def callback(indata, outdata, frames, time, status):
+    # buffer and states to global
+    global in_buffer, out_buffer, states_1, states_2
+    if status:
+        print(status)
+    # write to buffer
+    in_buffer[:-block_shift] = in_buffer[block_shift:]
+    in_buffer[-block_shift:] = np.squeeze(indata)
+    # calculate fft of input block
+    in_block_fft = np.fft.rfft(in_buffer)
+    in_mag = np.abs(in_block_fft)
+    in_phase = np.angle(in_block_fft)
+    # reshape magnitude to input dimensions
+    in_mag = np.reshape(in_mag, (1,1,-1)).astype('float32')
+    # set tensors to the first model
+    interpreter_1.set_tensor(input_details_1[1]['index'], states_1)
+    interpreter_1.set_tensor(input_details_1[0]['index'], in_mag)
+    # run calculation 
+    interpreter_1.invoke()
+    # get the output of the first block
+    out_mask = interpreter_1.get_tensor(output_details_1[0]['index']) 
+    states_1 = interpreter_1.get_tensor(output_details_1[1]['index'])   
+    # calculate the ifft
+    estimated_complex = in_mag * out_mask * np.exp(1j * in_phase)
+    estimated_block = np.fft.irfft(estimated_complex)
+    # reshape the time domain block
+    estimated_block = np.reshape(estimated_block, (1,1,-1)).astype('float32')
+    # set tensors to the second block
+    interpreter_2.set_tensor(input_details_2[1]['index'], states_2)
+    interpreter_2.set_tensor(input_details_2[0]['index'], estimated_block)
+    # run calculation
+    interpreter_2.invoke()
+    # get output tensors
+    out_block = interpreter_2.get_tensor(output_details_2[0]['index']) 
+    states_2 = interpreter_2.get_tensor(output_details_2[1]['index']) 
+    # write to buffer
+    out_buffer[:-block_shift] = out_buffer[block_shift:]
+    out_buffer[-block_shift:] = np.zeros((block_shift))
+    out_buffer  += np.squeeze(out_block)
+    # output to soundcard
+    outdata[:] = np.expand_dims(out_buffer[:block_shift], axis=-1)
+    
+
+
+try:
+    with sd.Stream(device=(args.input_device, args.output_device),
+                   samplerate=fs_target, blocksize=block_shift,
+                   dtype=np.float32, latency=args.latency,
+                   channels=1, callback=callback):
+        print('#' * 80)
+        print('press Return to quit')
+        print('#' * 80)
+        input()
+except KeyboardInterrupt:
+    parser.exit('')
+except Exception as e:
+    parser.exit(type(e).__name__ + ': ' + str(e))
+    

models/DTLN (yash-04)/real_time_processing.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Tue Jun 23 16:23:15 2020
+
+@author: nils
+"""
+
+import soundfile as sf
+import numpy as np
+import tensorflow as tf
+
+
+
+##########################
+# the values are fixed, if you need other values, you have to retrain.
+# The sampling rate of 16k is also fix.
+block_len = 512
+block_shift = 128
+# load model
+model = tf.saved_model.load('./pretrained_model/dtln_saved_model')
+infer = model.signatures["serving_default"]
+# load audio file at 16k fs (please change)
+audio,fs = sf.read('path_to_your_favorite_audio.wav')
+# check for sampling rate
+if fs != 16000:
+    raise ValueError('This model only supports 16k sampling rate.')
+# preallocate output audio
+out_file = np.zeros((len(audio)))
+# create buffer
+in_buffer = np.zeros((block_len))
+out_buffer = np.zeros((block_len))
+# calculate number of blocks
+num_blocks = (audio.shape[0] - (block_len-block_shift)) // block_shift
+# iterate over the number of blcoks        
+for idx in range(num_blocks):
+    # shift values and write to buffer
+    in_buffer[:-block_shift] = in_buffer[block_shift:]
+    in_buffer[-block_shift:] = audio[idx*block_shift:(idx*block_shift)+block_shift]
+    # create a batch dimension of one
+    in_block = np.expand_dims(in_buffer, axis=0).astype('float32')
+    # process one block
+    out_block= infer(tf.constant(in_block))['conv1d_1']
+    # shift values and write to buffer
+    out_buffer[:-block_shift] = out_buffer[block_shift:]
+    out_buffer[-block_shift:] = np.zeros((block_shift))
+    out_buffer  += np.squeeze(out_block)
+    # write block to output file
+    out_file[idx*block_shift:(idx*block_shift)+block_shift] = out_buffer[:block_shift]
+    
+    
+# write to .wav file 
+sf.write('out.wav', out_file, fs) 
+
+print('Processing finished.')

models/DTLN (yash-04)/real_time_processing_onnx.py

@@ -0,0 +1,106 @@
+"""
+This is an example how to implement real time processing of the DTLN ONNX
+model in python.
+
+Please change the name of the .wav file at line 49 before running the sript.
+For the ONNX runtime call: $ pip install onnxruntime
+    
+    
+
+Author: Nils L. Westhausen (nils.westhausen@uol.de)
+Version: 03.07.2020
+
+This code is licensed under the terms of the MIT-license.
+"""
+
+import soundfile as sf
+import numpy as np
+import time
+import onnxruntime
+
+
+
+##########################
+# the values are fixed, if you need other values, you have to retrain.
+# The sampling rate of 16k is also fix.
+block_len = 512
+block_shift = 128
+# load models
+interpreter_1 = onnxruntime.InferenceSession('./model_1.onnx')
+model_input_names_1 = [inp.name for inp in interpreter_1.get_inputs()]
+# preallocate input
+model_inputs_1 = {
+            inp.name: np.zeros(
+                [dim if isinstance(dim, int) else 1 for dim in inp.shape],
+                dtype=np.float32)
+            for inp in interpreter_1.get_inputs()}
+# load models
+interpreter_2 = onnxruntime.InferenceSession('./model_2.onnx')
+model_input_names_2 = [inp.name for inp in interpreter_2.get_inputs()]
+# preallocate input
+model_inputs_2 = {
+            inp.name: np.zeros(
+                [dim if isinstance(dim, int) else 1 for dim in inp.shape],
+                dtype=np.float32)
+            for inp in interpreter_2.get_inputs()}
+
+# load audio file
+audio,fs = sf.read('path/to/your/favorite.wav')
+# check for sampling rate
+if fs != 16000:
+    raise ValueError('This model only supports 16k sampling rate.')
+# preallocate output audio
+out_file = np.zeros((len(audio)))
+# create buffer
+in_buffer = np.zeros((block_len)).astype('float32')
+out_buffer = np.zeros((block_len)).astype('float32')
+# calculate number of blocks
+num_blocks = (audio.shape[0] - (block_len-block_shift)) // block_shift
+# iterate over the number of blcoks  
+time_array = []      
+for idx in range(num_blocks):
+    start_time = time.time()
+    # shift values and write to buffer
+    in_buffer[:-block_shift] = in_buffer[block_shift:]
+    in_buffer[-block_shift:] = audio[idx*block_shift:(idx*block_shift)+block_shift]
+    # calculate fft of input block
+    in_block_fft = np.fft.rfft(in_buffer)
+    in_mag = np.abs(in_block_fft)
+    in_phase = np.angle(in_block_fft)
+    # reshape magnitude to input dimensions
+    in_mag = np.reshape(in_mag, (1,1,-1)).astype('float32')
+    # set block to input
+    model_inputs_1[model_input_names_1[0]] = in_mag
+    # run calculation 
+    model_outputs_1 = interpreter_1.run(None, model_inputs_1)
+    # get the output of the first block
+    out_mask = model_outputs_1[0]
+    # set out states back to input
+    model_inputs_1[model_input_names_1[1]] = model_outputs_1[1]  
+    # calculate the ifft
+    estimated_complex = in_mag * out_mask * np.exp(1j * in_phase)
+    estimated_block = np.fft.irfft(estimated_complex)
+    # reshape the time domain block
+    estimated_block = np.reshape(estimated_block, (1,1,-1)).astype('float32')
+    # set tensors to the second block
+    # interpreter_2.set_tensor(input_details_1[1]['index'], states_2)
+    model_inputs_2[model_input_names_2[0]] = estimated_block
+    # run calculation
+    model_outputs_2 = interpreter_2.run(None, model_inputs_2)
+    # get output
+    out_block = model_outputs_2[0]
+    # set out states back to input
+    model_inputs_2[model_input_names_2[1]] = model_outputs_2[1]
+    # shift values and write to buffer
+    out_buffer[:-block_shift] = out_buffer[block_shift:]
+    out_buffer[-block_shift:] = np.zeros((block_shift))
+    out_buffer  += np.squeeze(out_block)
+    # write block to output file
+    out_file[idx*block_shift:(idx*block_shift)+block_shift] = out_buffer[:block_shift]
+    time_array.append(time.time()-start_time)
+    
+# write to .wav file 
+sf.write('out.wav', out_file, fs) 
+print('Processing Time [ms]:')
+print(np.mean(np.stack(time_array))*1000)
+print('Processing finished.')

models/DTLN (yash-04)/real_time_processing_tf_lite.py

@@ -0,0 +1,102 @@
+"""
+This is an example how to implement real time processing of the DTLN tf light
+model in python.
+
+Please change the name of the .wav file at line 43 before running the sript.
+For .whl files of the tf light runtime go to: 
+    https://www.tensorflow.org/lite/guide/python
+    
+Author: Nils L. Westhausen (nils.westhausen@uol.de)
+Version: 30.06.2020
+
+This code is licensed under the terms of the MIT-license.
+"""
+
+import soundfile as sf
+import numpy as np
+import tflite_runtime.interpreter as tflite
+import time
+
+
+
+##########################
+# the values are fixed, if you need other values, you have to retrain.
+# The sampling rate of 16k is also fix.
+block_len = 512
+block_shift = 128
+# load models
+interpreter_1 = tflite.Interpreter(model_path='./pretrained_model/model_1.tflite')
+interpreter_1.allocate_tensors()
+interpreter_2 = tflite.Interpreter(model_path='./pretrained_model/model_2.tflite')
+interpreter_2.allocate_tensors()
+
+# Get input and output tensors.
+input_details_1 = interpreter_1.get_input_details()
+output_details_1 = interpreter_1.get_output_details()
+
+input_details_2 = interpreter_2.get_input_details()
+output_details_2 = interpreter_2.get_output_details()
+# create states for the lstms
+states_1 = np.zeros(input_details_1[1]['shape']).astype('float32')
+states_2 = np.zeros(input_details_2[1]['shape']).astype('float32')
+# load audio file at 16k fs (please change)
+audio,fs = sf.read('path/to/your/favorite/.wav')
+# check for sampling rate
+if fs != 16000:
+    raise ValueError('This model only supports 16k sampling rate.')
+# preallocate output audio
+out_file = np.zeros((len(audio)))
+# create buffer
+in_buffer = np.zeros((block_len)).astype('float32')
+out_buffer = np.zeros((block_len)).astype('float32')
+# calculate number of blocks
+num_blocks = (audio.shape[0] - (block_len-block_shift)) // block_shift
+time_array = []      
+# iterate over the number of blcoks  
+for idx in range(num_blocks):
+    start_time = time.time()
+    # shift values and write to buffer
+    in_buffer[:-block_shift] = in_buffer[block_shift:]
+    in_buffer[-block_shift:] = audio[idx*block_shift:(idx*block_shift)+block_shift]
+    # calculate fft of input block
+    in_block_fft = np.fft.rfft(in_buffer)
+    in_mag = np.abs(in_block_fft)
+    in_phase = np.angle(in_block_fft)
+    # reshape magnitude to input dimensions
+    in_mag = np.reshape(in_mag, (1,1,-1)).astype('float32')
+    # set tensors to the first model
+    interpreter_1.set_tensor(input_details_1[1]['index'], states_1)
+    interpreter_1.set_tensor(input_details_1[0]['index'], in_mag)
+    # run calculation 
+    interpreter_1.invoke()
+    # get the output of the first block
+    out_mask = interpreter_1.get_tensor(output_details_1[0]['index']) 
+    states_1 = interpreter_1.get_tensor(output_details_1[1]['index'])   
+    # calculate the ifft
+    estimated_complex = in_mag * out_mask * np.exp(1j * in_phase)
+    estimated_block = np.fft.irfft(estimated_complex)
+    # reshape the time domain block
+    estimated_block = np.reshape(estimated_block, (1,1,-1)).astype('float32')
+    # set tensors to the second block
+    interpreter_2.set_tensor(input_details_2[1]['index'], states_2)
+    interpreter_2.set_tensor(input_details_2[0]['index'], estimated_block)
+    # run calculation
+    interpreter_2.invoke()
+    # get output tensors
+    out_block = interpreter_2.get_tensor(output_details_2[0]['index']) 
+    states_2 = interpreter_2.get_tensor(output_details_2[1]['index']) 
+    
+    
+    # shift values and write to buffer
+    out_buffer[:-block_shift] = out_buffer[block_shift:]
+    out_buffer[-block_shift:] = np.zeros((block_shift))
+    out_buffer  += np.squeeze(out_block)
+    # write block to output file
+    out_file[idx*block_shift:(idx*block_shift)+block_shift] = out_buffer[:block_shift]
+    time_array.append(time.time()-start_time)
+    
+# write to .wav file 
+sf.write('out.wav', out_file, fs) 
+print('Processing Time [ms]:')
+print(np.mean(np.stack(time_array))*1000)
+print('Processing finished.')

models/DTLN (yash-04)/run_evaluation.py

@@ -0,0 +1,131 @@
+# -*- coding: utf-8 -*-
+"""
+Script to process a folder of .wav files with a trained DTLN model. 
+This script supports subfolders and names the processed files the same as the 
+original. The model expects 16kHz audio .wav files. Files with other 
+sampling rates will be resampled. Stereo files will be downmixed to mono.
+
+The idea of this script is to use it for baseline or comparison purpose.
+
+Example call:
+    $python run_evaluation.py -i /name/of/input/folder  \
+                              -o /name/of/output/folder \
+                              -m /name/of/the/model.h5
+
+Author: Nils L. Westhausen (nils.westhausen@uol.de)
+Version: 13.05.2020
+
+This code is licensed under the terms of the MIT-license.
+"""
+
+import soundfile as sf
+import librosa
+import numpy as np
+import os
+import argparse
+from DTLN_model import DTLN_model
+
+
+
+def process_file(model, audio_file_name, out_file_name):
+    '''
+    Funtion to read an audio file, rocess it by the network and write the 
+    enhanced audio to .wav file.
+
+    Parameters
+    ----------
+    model : Keras model
+        Keras model, which accepts audio in the size (1,timesteps).
+    audio_file_name : STRING
+        Name and path of the input audio file.
+    out_file_name : STRING
+        Name and path of the target file.
+
+    '''
+    
+    # read audio file with librosa to handle resampling and enforce mono
+    in_data,fs = librosa.core.load(audio_file_name, sr=16000, mono=True)
+    # get length of file
+    len_orig = len(in_data)
+    # pad audio
+    zero_pad = np.zeros(384)
+    in_data = np.concatenate((zero_pad, in_data, zero_pad), axis=0)
+    # predict audio with the model
+    predicted = model.predict_on_batch(
+        np.expand_dims(in_data,axis=0).astype(np.float32))
+    # squeeze the batch dimension away
+    predicted_speech = np.squeeze(predicted)
+    predicted_speech = predicted_speech[384:384+len_orig]
+    # write the file to target destination
+    sf.write(out_file_name, predicted_speech,fs)
+      
+
+def process_folder(model, folder_name, new_folder_name):
+    '''
+    Function to find .wav files in the folder and subfolders of "folder_name",
+    process each .wav file with an algorithm and write it back to disk in the 
+    folder "new_folder_name". The structure of the original directory is 
+    preserved. The processed files will be saved with the same name as the 
+    original file.
+
+    Parameters
+    ----------
+    model : Keras model
+        Keras model, which accepts audio in the size (1,timesteps).
+    folder_name : STRING
+        Input folder with .wav files.
+    new_folder_name : STRING
+        Traget folder for the processed files.
+
+    '''
+    
+    # empty list for file and folder names
+    file_names = [];
+    directories = [];
+    new_directories = [];
+    # walk through the directory
+    for root, dirs, files in os.walk(folder_name):
+        for file in files:
+            # look for .wav files
+            if file.endswith(".wav"):
+                # write paths and filenames to lists
+                file_names.append(file)
+                directories.append(root)
+                # create new directory names
+                new_directories.append(root.replace(folder_name, new_folder_name))
+                # check if the new directory already exists, if not create it
+                if not os.path.exists(root.replace(folder_name, new_folder_name)):
+                    os.makedirs(root.replace(folder_name, new_folder_name))
+    # iterate over all .wav files             
+    for idx in range(len(file_names)):
+        # process each file with the model
+        process_file(model, os.path.join(directories[idx],file_names[idx]), 
+                     os.path.join(new_directories[idx],file_names[idx]))
+        print(file_names[idx] + ' processed successfully!')
+    
+           
+
+
+if __name__ == '__main__':
+    # arguement parser for running directly from the command line
+    parser = argparse.ArgumentParser(description='data evaluation')
+    parser.add_argument('--in_folder', '-i',
+                        help='folder with input files')
+    parser.add_argument('--out_folder', '-o',
+                        help='target folder for processed files')
+    parser.add_argument('--model', '-m',
+                        help='weights of the enhancement model in .h5 format')
+    args = parser.parse_args()
+    # determine type of model
+    if args.model.find('_norm_') != -1:
+        norm_stft = True
+    else:
+        norm_stft = False
+    # create class instance
+    modelClass = DTLN_model();
+    # build the model in default configuration
+    modelClass.build_DTLN_model(norm_stft=norm_stft)
+    # load weights of the .h5 file
+    modelClass.model.load_weights(args.model)
+    # process the folder
+    process_folder(modelClass.model, args.in_folder, args.out_folder)

models/DTLN (yash-04)/run_training.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Script to train the DTLN model in default settings. The folders for noisy and
+clean files are expected to have the same number of files and the files to 
+have the same name. The training procedure always saves the best weights of 
+the model into the folder "./models_'runName'/". Also a log file of the 
+training progress is written there. To change any parameters go to the 
+"DTLN_model.py" file or use "modelTrainer.parameter = XY" in this file.
+It is recommended to run the training on a GPU. The setup is optimized for the
+DNS-Challenge data set. If you use a custom data set, just play around with
+the parameters.
+
+Please change the folder names before starting the training. 
+
+Example call:
+    $python run_training.py
+
+Author: Nils L. Westhausen (nils.westhausen@uol.de)
+Version: 13.05.2020
+
+This code is licensed under the terms of the MIT-license.
+"""
+
+from DTLN_model import DTLN_model
+import os
+
+# use the GPU with idx 0
+os.environ["CUDA_VISIBLE_DEVICES"]='0'
+# activate this for some reproducibility
+os.environ['TF_DETERMINISTIC_OPS'] = '1'
+
+
+
+
+
+# path to folder containing the noisy or mixed audio training files
+path_to_train_mix = '/path/to/noisy/training/data/'
+# path to folder containing the clean/speech files for training
+path_to_train_speech = '/path/to/clean/training/data/'
+# path to folder containing the noisy or mixed audio validation data
+path_to_val_mix = '/path/to/noisy/validation/data/'
+# path to folder containing the clean audio validation data
+path_to_val_speech = '/path/to/clean/validation/data/'
+
+# name your training run
+runName = 'DTLN_model'
+# create instance of the DTLN model class
+modelTrainer = DTLN_model()
+# build the model
+modelTrainer.build_DTLN_model()
+# compile it with optimizer and cost function for training
+modelTrainer.compile_model()
+# train the model
+modelTrainer.train_model(runName, path_to_train_mix, path_to_train_speech, \
+                         path_to_val_mix, path_to_val_speech)
+
+
+

models/DTLN (yash-04)/source.txt

@@ -0,0 +1 @@
+https://huggingface.co/yash-04/DTLN

models/DTLN (yash-04)/tflite_env.yml

@@ -0,0 +1,73 @@
+name: tflite-env
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - audioread=2.1.8=py37hc8dfbb8_2
+  - bzip2=1.0.8=h0b31af3_2
+  - ca-certificates=2020.6.20=hecda079_0
+  - certifi=2020.6.20=py37hc8dfbb8_0
+  - cffi=1.14.0=py37hc512035_1
+  - cycler=0.10.0=py_2
+  - decorator=4.4.2=py_0
+  - ffmpeg=4.2.3=hd0c0d6a_0
+  - freetype=2.10.2=h8da9a1a_0
+  - gettext=0.19.8.1=h1f1d5ed_1
+  - gmp=6.2.0=h4a8c4bd_2
+  - gnutls=3.6.13=hc269f14_0
+  - joblib=0.15.1=py_0
+  - kiwisolver=1.2.0=py37ha1cc60f_0
+  - lame=3.100=h1de35cc_1001
+  - libblas=3.8.0=17_openblas
+  - libcblas=3.8.0=17_openblas
+  - libcxx=10.0.0=1
+  - libedit=3.1.20191231=haf1e3a3_0
+  - libffi=3.3=h0a44026_1
+  - libflac=1.3.3=h4a8c4bd_0
+  - libgfortran=4.0.0=2
+  - libiconv=1.15=h0b31af3_1006
+  - liblapack=3.8.0=17_openblas
+  - libllvm8=8.0.1=h770b8ee_0
+  - libogg=1.3.2=h0b31af3_1002
+  - libopenblas=0.3.10=h3d69b6c_0
+  - libpng=1.6.37=hbbe82c9_1
+  - librosa=0.7.2=py_1
+  - libsndfile=1.0.28=h4a8c4bd_1000
+  - libvorbis=1.3.6=h4a8c4bd_2
+  - llvm-openmp=10.0.0=h28b9765_0
+  - llvmlite=0.31.0=py37hb548287_1
+  - matplotlib-base=3.2.2=py37hddda452_0
+  - ncurses=6.2=h0a44026_1
+  - nettle=3.4.1=h3efe00b_1002
+  - numba=0.48.0=py37h4f17bb1_0
+  - numpy=1.18.5=py37h7687784_0
+  - openh264=2.1.1=hd174df1_0
+  - openssl=1.1.1g=h0b31af3_0
+  - pip=20.1.1=py37_1
+  - portaudio=19.6.0=h647c56a_4
+  - pycparser=2.20=pyh9f0ad1d_2
+  - pyparsing=2.4.7=pyh9f0ad1d_0
+  - pysoundfile=0.10.2=py_1001
+  - python=3.7.7=hf48f09d_4
+  - python-dateutil=2.8.1=py_0
+  - python-sounddevice=0.3.15=pyh8c360ce_0
+  - python_abi=3.7=1_cp37m
+  - readline=8.0=h1de35cc_0
+  - resampy=0.2.2=py_0
+  - scikit-learn=0.23.1=py37hf5857e7_0
+  - scipy=1.5.0=py37hce1b9e5_0
+  - setuptools=47.3.1=py37_0
+  - six=1.15.0=pyh9f0ad1d_0
+  - sqlite=3.32.3=hffcf06c_0
+  - threadpoolctl=2.1.0=pyh5ca1d4c_0
+  - tk=8.6.10=hb0a8c7a_0
+  - tornado=6.0.4=py37h9bfed18_1
+  - wheel=0.34.2=py37_0
+  - x264=1!152.20180806=h1de35cc_0
+  - xz=5.2.5=h1de35cc_0
+  - zlib=1.2.11=h1de35cc_3
+  - pip:
+    - flatbuffers==1.12
+    - tflite-runtime==2.1.0.post1
+prefix: /Applications/anaconda3/envs/tflite-env
+

models/DTLN (yash-04)/train_env.yml

@@ -0,0 +1,120 @@
+name: train_env
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=main
+  - _tflow_select=2.1.0=gpu
+  - absl-py=0.9.0=py37_0
+  - astunparse=1.6.3=py_0
+  - audioread=2.1.8=py37hc8dfbb8_2
+  - blas=1.0=mkl
+  - blinker=1.4=py37_0
+  - bzip2=1.0.8=h516909a_2
+  - c-ares=1.15.0=h7b6447c_1001
+  - ca-certificates=2020.4.5.2=hecda079_0
+  - cachetools=4.1.0=py_1
+  - certifi=2020.4.5.2=py37hc8dfbb8_0
+  - cffi=1.14.0=py37he30daa8_1
+  - chardet=3.0.4=py37_1003
+  - click=7.1.2=py_0
+  - cryptography=2.9.2=py37h1ba5d50_0
+  - cudatoolkit=10.1.243=h6bb024c_0
+  - cudnn=7.6.5=cuda10.1_0
+  - cupti=10.1.168=0
+  - cycler=0.10.0=py_2
+  - decorator=4.4.2=py_0
+  - ffmpeg=4.2.3=h167e202_0
+  - freetype=2.10.2=he06d7ca_0
+  - gast=0.3.3=py_0
+  - gettext=0.19.8.1=h5e8e0c9_1
+  - gmp=6.2.0=he1b5a44_2
+  - gnutls=3.6.13=h79a8f9a_0
+  - google-auth=1.14.1=py_0
+  - google-auth-oauthlib=0.4.1=py_2
+  - google-pasta=0.2.0=py_0
+  - grpcio=1.27.2=py37hf8bcb03_0
+  - h5py=2.10.0=py37h7918eee_0
+  - hdf5=1.10.4=hb1b8bf9_0
+  - icu=58.2=hf484d3e_1000
+  - idna=2.9=py_1
+  - intel-openmp=2020.1=217
+  - joblib=0.15.1=py_0
+  - keras-preprocessing=1.1.0=py_1
+  - kiwisolver=1.2.0=py37h99015e2_0
+  - lame=3.100=h14c3975_1001
+  - ld_impl_linux-64=2.33.1=h53a641e_7
+  - libedit=3.1.20181209=hc058e9b_0
+  - libffi=3.3=he6710b0_1
+  - libflac=1.3.3=he1b5a44_0
+  - libgcc-ng=9.1.0=hdf63c60_0
+  - libgfortran-ng=7.3.0=hdf63c60_0
+  - libiconv=1.15=h516909a_1006
+  - libllvm8=8.0.1=hc9558a2_0
+  - libogg=1.3.2=h516909a_1002
+  - libpng=1.6.37=hed695b0_1
+  - libprotobuf=3.12.3=hd408876_0
+  - librosa=0.7.2=py_1
+  - libsndfile=1.0.28=he1b5a44_1000
+  - libstdcxx-ng=9.1.0=hdf63c60_0
+  - libvorbis=1.3.6=he1b5a44_2
+  - llvmlite=0.31.0=py37h5202443_1
+  - markdown=3.1.1=py37_0
+  - matplotlib-base=3.1.3=py37hef1b27d_0
+  - mkl=2020.1=217
+  - mkl-service=2.3.0=py37he904b0f_0
+  - mkl_fft=1.0.15=py37ha843d7b_0
+  - mkl_random=1.1.1=py37h0573a6f_0
+  - ncurses=6.2=he6710b0_1
+  - nettle=3.4.1=h1bed415_1002
+  - numba=0.48.0=py37hb3f55d8_0
+  - numpy=1.18.1=py37h4f9e942_0
+  - numpy-base=1.18.1=py37hde5b4d6_1
+  - oauthlib=3.1.0=py_0
+  - openh264=2.1.1=h8b12597_0
+  - openssl=1.1.1g=h516909a_0
+  - opt_einsum=3.1.0=py_0
+  - pip=20.0.2=py37_3
+  - protobuf=3.12.3=py37he6710b0_0
+  - pyasn1=0.4.8=py_0
+  - pyasn1-modules=0.2.7=py_0
+  - pycparser=2.20=py_0
+  - pyjwt=1.7.1=py37_0
+  - pyopenssl=19.1.0=py37_0
+  - pyparsing=2.4.7=pyh9f0ad1d_0
+  - pysocks=1.7.1=py37_0
+  - pysoundfile=0.10.2=py_1001
+  - python=3.7.7=hcff3b4d_5
+  - python-dateutil=2.8.1=py_0
+  - python_abi=3.7=1_cp37m
+  - readline=8.0=h7b6447c_0
+  - requests=2.23.0=py37_0
+  - requests-oauthlib=1.3.0=py_0
+  - resampy=0.2.2=py_0
+  - rsa=4.0=py_0
+  - scikit-learn=0.22.1=py37hd81dba3_0
+  - scipy=1.4.1=py37h0b6359f_0
+  - setuptools=47.1.1=py37_0
+  - six=1.15.0=py_0
+  - sqlite=3.31.1=h62c20be_1
+  - tensorboard=2.2.1=pyh532a8cf_0
+  - tensorboard-plugin-wit=1.6.0=py_0
+  - tensorflow=2.2.0=gpu_py37h1a511ff_0
+  - tensorflow-base=2.2.0=gpu_py37h8a81be8_0
+  - tensorflow-estimator=2.2.0=pyh208ff02_0
+  - tensorflow-gpu=2.2.0=h0d30ee6_0
+  - termcolor=1.1.0=py37_1
+  - tk=8.6.8=hbc83047_0
+  - tornado=6.0.4=py37h8f50634_1
+  - urllib3=1.25.8=py37_0
+  - werkzeug=1.0.1=py_0
+  - wheel=0.34.2=py37_0
+  - wrapt=1.12.1=py37h7b6447c_1
+  - x264=1!152.20180806=h14c3975_0
+  - xz=5.2.5=h7b6447c_0
+  - zlib=1.2.11=h7b6447c_3
+  - pip:
+    - lxml==4.5.1
+    - wavinfo==1.5
+prefix: /home/nils/anaconda3/envs/tfenv
+