visualize_per_sentence.py

# PREREQUISITY

# correct_figure.py -> makes analytic.pkl & CER -> per sentence No Audinterface sliding window
import pandas as pd
import os
import numpy as np
from pathlib import Path
import matplotlib.pyplot as plt
import audiofile

columns = ['prompt-arousal', 
           'prompt-dominance', 
           'prompt-valence', 
           'prompt-Angry',
           'prompt-Sad', 
           'prompt-Happy', 
           'prompt-Surprise', 
           'prompt-Fear',
           'prompt-Disgust', 
           'prompt-Contempt', 
           'prompt-Neutral',
           'styletts2-arousal', 
           'styletts2-dominance', 
           'styletts2-valence',
           'styletts2-Angry', 
           'styletts2-Sad', 
           'styletts2-Happy',
           'styletts2-Surprise', 
           'styletts2-Fear', 
           'styletts2-Disgust',
           'styletts2-Contempt', 
           'styletts2-Neutral', 
           'cer-prompt',
           'cer-styletts2']

FULL_PKL = ['english_4x_analytic.pkl',
         'english_analytic.pkl',
         'foreign_4x_analytic.pkl',
         'foreign_analytic.pkl',
         'human_analytic.pkl']
# -------------------------------------------



LABELS = ['arousal', 'dominance', 'valence',
        #    'speech_synthesizer', 'synthetic_singing',
           'Angry',
           'Sad',
           'Happy',
           'Surprise', 
            'Fear', 
            'Disgust', 
            'Contempt', 
            'Neutral'
            ]




    # https://arxiv.org/pdf/2407.12229
    #  https://arxiv.org/pdf/2312.05187
    # https://arxiv.org/abs/2407.05407
    # https://arxiv.org/pdf/2408.06577
    # https://arxiv.org/pdf/2309.07405
preds  = {}

for file_interface in FULL_PKL:
    y = pd.read_pickle(file_interface)
    # y = y.rolling(20).mean()[19:]  --> avoid when printing character error rate
    preds[file_interface] = y #.sort_values('styletts2-valence')
    print(f'\n\n         {file_interface}\n_____________________________\n', 
          f"{y['cer-prompt'].mean()=}", 
          f"{y['cer-styletts2'].mean()=}\n\n")
    


# =================================== cER ---------------------------


for lang in ['english', 
             'foreign']:
            

            fig, ax = plt.subplots(nrows=8, ncols=2, figsize=(24,20.7),
                                   gridspec_kw={'hspace': 0, 'wspace': .04})


            

            time_stamp = np.arange(len(preds['english_analytic.pkl']))
            _z = np.zeros(len(preds['english_analytic.pkl']))
            for j, dim in enumerate(['arousal', 'dominance', 'valence']):

                # MIMIC3                      

                ax[j, 0].plot(time_stamp, preds[f'{lang}_analytic.pkl'][f'styletts2-{dim}'], 
                            color=(0,104/255,139/255), 
                            label='mean_1', 
                            linewidth=2)
                ax[j, 0].fill_between(time_stamp,

                                _z,
                                preds['human_analytic.pkl'][f'styletts2-{dim}'],

                                color=(.2,.2,.2), 
                                alpha=0.244)
                if j == 0:
                    if lang == 'english':
                        desc = 'English'
                    else:
                        desc = 'Non-English'
                    ax[j, 0].legend([f'StyleTTS2 using Mimic-3 {desc}',
                                     f'StyleTTS2 uising EmoDB'], 
                                    prop={'size': 14},
                                    )
                ax[j, 0].set_ylabel(dim.lower(), color=(.4, .4, .4), fontsize=17)
                
                # TICK
                ax[j, 0].set_ylim([1e-7, .9999])
                # ax[j, 0].set_yticks([.25, .5,.75])
                # ax[j, 0].set_yticklabels(['0.25', '.5', '0.75'])
                ax[j, 0].set_xticklabels(['' for _ in ax[j, 0].get_xticklabels()])
                ax[j, 0].set_xlim([time_stamp[0], time_stamp[-1]])


            # MIMIC3   4x speed


                ax[j, 1].plot(time_stamp, preds[f'{lang}_4x_analytic.pkl'][f'styletts2-{dim}'],
                            color=(0,104/255,139/255), 
                            label='mean_1', 
                            linewidth=2)
                ax[j, 1].fill_between(time_stamp,

                                _z,
                                preds['human_analytic.pkl'][f'styletts2-{dim}'],

                                color=(.2,.2,.2),
                                alpha=0.244)
                if j == 0:
                    if lang == 'english':
                        desc = 'English'
                    else:
                        desc = 'Non-English'                 
                    ax[j, 1].legend([f'StyleTTS2 using Mimic-3 {desc} 4x speed',
                                    f'StyleTTS2 using EmoDB'], 
                                    prop={'size': 14}, 
                                    #  loc='lower right'
                                    )


                ax[j, 1].set_xlabel('720 Harvard Sentences')



                # TICK
                ax[j, 1].set_ylim([1e-7, .9999])
                # ax[j, 1].set_yticklabels(['' for _ in ax[j, 1].get_yticklabels()])
                ax[j, 1].set_xticklabels(['' for _ in ax[j, 0].get_xticklabels()])
                ax[j, 1].set_xlim([time_stamp[0], time_stamp[-1]])




                ax[j, 0].grid()
                ax[j, 1].grid()
            # CATEGORIE





            
            for j, dim in enumerate(['Angry', 
                                    'Sad',
                                    'Happy',
                                    #  'Surprise', 
                                    'Fear',
                                    'Disgust', 
                                    #  'Contempt',
                                    #  'Neutral'
                                    ]):   # ASaHSuFDCN
                j = j + 3  # skip A/D/V suplt                         

                # MIMIC3                      

                ax[j, 0].plot(time_stamp, preds[f'{lang}_analytic.pkl'][f'styletts2-{dim}'], 
                            color=(0,104/255,139/255), 
                            label='mean_1', 
                            linewidth=2)
                ax[j, 0].fill_between(time_stamp,

                                _z,
                                preds['human_analytic.pkl'][f'styletts2-{dim}'],

                                color=(.2,.2,.2), 
                                alpha=0.244)
                # ax[j, 0].legend(['StyleTTS2 style mimic3',
                #                  'StyleTTS2 style crema-d'], 
                #                  prop={'size': 10}, 
                #                 #  loc='upper left'
                # )


                ax[j, 0].set_ylabel(dim.lower(), color=(.4, .4, .4), fontsize=17)

                # TICKS
                ax[j, 0].set_ylim([1e-7, .9999])
                ax[j, 0].set_xlim([time_stamp[0], time_stamp[-1]])
                ax[j, 0].set_xticklabels(['' for _ in ax[j, 0].get_xticklabels()])
                ax[j, 0].set_xlabel('720 Harvard Sentences', fontsize=17, color=(.2,.2,.2))


            # MIMIC3   4x speed


                ax[j, 1].plot(time_stamp, preds[f'{lang}_4x_analytic.pkl'][f'styletts2-{dim}'],
                            color=(0,104/255,139/255), 
                            label='mean_1', 
                            linewidth=2)
                ax[j, 1].fill_between(time_stamp,

                                _z,
                                preds['human_analytic.pkl'][f'styletts2-{dim}'],

                                color=(.2,.2,.2), 
                                alpha=0.244)
                # ax[j, 1].legend(['StyleTTS2 style mimic3   4x speed',
                #                  'StyleTTS2 style crema-d'], 
                #                  prop={'size': 10},
                #                 #  loc='upper left'
                # )
                ax[j, 1].set_xlabel('720 Harvard Sentences', fontsize=17, color=(.2,.2,.2))
                ax[j, 1].set_ylim([1e-7, .9999])
                # ax[j, 1].set_yticklabels(['' for _ in ax[j, 1].get_yticklabels()])
                ax[j, 1].set_xticklabels(['' for _ in ax[j, 1].get_xticklabels()])
                ax[j, 1].set_xlim([time_stamp[0], time_stamp[-1]])
                





                ax[j, 0].grid()
                ax[j, 1].grid()



            plt.savefig(f'persentence_{lang}.pdf', bbox_inches='tight')
            plt.close()