mimic3 prompt scripts

engineer_foreign_style_vectors.py

@@ -0,0 +1,186 @@
+# create foreign style vectors - to use for english StyleTTS2
+
+# It may crash due to non-truly-blocking shutil.copyfile() saying onnx protobuf incomplete file
+# You have to rerun the script - it will copy all voices from hf:mimic3-voices to ~/.local/mimic3
+from pathlib import Path
+import shutil
+import csv
+import io
+import os
+import typing
+import wave
+import sys
+from mimic3_tts.__main__ import (CommandLineInterfaceState,
+                                 get_args,
+                                 initialize_args,
+                                 initialize_tts,
+                                 # print_voices,
+                                 # process_lines,
+                                 shutdown_tts,
+                                 OutputNaming,
+                                 process_line)
+import time
+ROOT_DIR = '/data/dkounadis/mimic3-voices/'
+
+foreign_voices = []
+for lang in os.listdir(ROOT_DIR + 'voices'):
+        if 'en_' not in lang:
+            for voice in os.listdir(ROOT_DIR + 'voices/' + lang):
+                # print('______\n', voice)
+                try:
+                    with open(ROOT_DIR + 'voices/' + lang + '/' + voice + '/speakers.txt', 'r') as f:
+                        foreign_voices += [lang + '/' + voice + '#' + spk.rstrip() for spk in f]
+                except FileNotFoundError:
+                    # spk = None  # siwis_low [has no speakers]
+                    foreign_voices.append(lang + '/' + voice)
+                # print(spk)
+                # print(os.listdir(ROOT_DIR + 'voices/' + lang + '/' + voice))
+# --- Now we have all speakers per voices -- so can we call mimic3 on those perhaps with copyfile
+# print(foreign_voices, len(foreign_voices))
+
+
+
+
+
+
+# ----------------------
+# print(foreign_voices.keys(), len(foreign_voices))
+# raise SystemExit
+
+
+def process_lines(state: CommandLineInterfaceState, wav_path=None):
+    '''MIMIC3 INTERNAL CALL that yields the sigh sound'''
+
+    args = state.args
+
+    result_idx = 0
+    print(f'why waitings in the for loop LIN {state.texts=}\n')
+    for line in state.texts:
+        # print(f'LIN {line=}\n')  # prints \n so is empty not getting the predifne text of state.texts
+        line_voice: typing.Optional[str] = None
+        line_id = ""
+        line = line.strip()
+        # if not line:
+        #     continue
+
+        if args.output_naming == OutputNaming.ID:
+            # Line has the format id|text instead of just text
+            with io.StringIO(line) as line_io:
+                reader = csv.reader(line_io, delimiter=args.csv_delimiter)
+                row = next(reader)
+                line_id, line = row[0], row[-1]
+                if args.csv_voice:
+                    line_voice = row[1]
+
+        process_line(line, state, line_id=line_id, line_voice=line_voice)
+        result_idx += 1
+    time.sleep(4)
+    # Write combined audio to stdout
+    if state.all_audio:
+        # _LOGGER.debug("Writing WAV audio to stdout")
+
+        if sys.stdout.isatty() and (not state.args.stdout):
+            with io.BytesIO() as wav_io:
+                wav_file_play: wave.Wave_write = wave.open(wav_io, "wb")
+                with wav_file_play:
+                    wav_file_play.setframerate(state.sample_rate_hz)
+                    wav_file_play.setsampwidth(state.sample_width_bytes)
+                    wav_file_play.setnchannels(state.num_channels)
+                    wav_file_play.writeframes(state.all_audio)
+
+                    # play_wav_bytes(state.args, wav_io.getvalue())
+                # wav_path = '_direct_call_2.wav'
+                with open(wav_path, 'wb') as wav_file:
+                    wav_file.write(wav_io.getvalue())
+                    wav_file.seek(0)
+                    print('\n\nTTSING', wav_path)
+    else:
+        print('\n\nDOES NOT TTSING --> ADD SOME time.sleep(4)', wav_path)                    
+
+# -----------------------------------------------------------------------------
+# cat _tmp_ssml.txt | mimic3 --cuda --ssml --noise-w 0.90001 --length-scale 0.91 --noise-scale 0.04 > noise_w=0.90_en_happy_2.wav
+# ======================================================================
+
+reference_wav_directory = 'style_vectors_speed1_ICASSP/' #out_dir # + '/wavs/style_vector_v2/'
+Path(reference_wav_directory).mkdir(parents=True, exist_ok=True)
+wav_dir = 'assets/wavs/'
+Path(wav_dir).mkdir(parents=True, exist_ok=True)
+for _id, _voice in enumerate(foreign_voices):
+
+    # Mimic3 GitHub Quota exceded:
+    #    https://github.com/MycroftAI/mimic3-voices
+    #    Above repo can exceed download quota of LFS
+    # Copy mimic-voices from local copies
+    #    clone https://huggingface.co/mukowaty/mimic3-voices/tree/main/voices
+    #    copy to ~/
+    # 
+    #
+    home_voice_dir = f'/home/audeering.local/dkounadis/.local/share/mycroft/mimic3/voices/{_voice.split("#")[0]}/'
+    Path(home_voice_dir).mkdir(parents=True, exist_ok=True)
+    speaker_free_voice_name = _voice.split("#")[0] if '#' in _voice else _voice
+    if (
+        (not os.path.isfile(home_voice_dir + 'generator.onnx')) or 
+        (os.path.getsize(home_voice_dir + 'generator.onnx') < 500)  # .onnx - is just LFS header
+               ):
+
+        # Copy
+
+        shutil.copyfile(
+            f'/data/dkounadis/mimic3-voices/voices/{speaker_free_voice_name}/generator.onnx',
+            home_voice_dir + 'generator.onnx')
+
+
+    prepare_file = _voice.replace('/', '_').replace('#', '_').replace('_low', '')
+    if 'cmu-arctic' in prepare_file:
+        prepare_file = prepare_file.replace('cmu-arctic', 'cmu_arctic') + '.wav'
+    else:
+        prepare_file = prepare_file + '.wav' # [...cmu-arctic...](....cmu_arctic....wav) 
+
+    # file_true = prepare_file.split('.wav')[0] + '_true_.wav'
+    # file_false = prepare_file.split('.wav')[0] + '_false_.wav'
+    # print(prepare_file, file_false, file_true)
+
+
+    reference_wav = reference_wav_directory + prepare_file
+    if not os.path.isfile(reference_wav):
+        
+        rate = 1  # high speed sounds nice if used as speaker-reference audio for StyleTTS2
+        _ssml = (
+            '<speak>'
+            '<prosody volume=\'64\'>'
+            f'<prosody rate=\'{rate}\'>'
+            f'<voice name=\'{_voice}\'>'
+            '<s>'
+            'Sweet dreams are made of this, .. !!! # I travel the world and the seven seas.'
+            '</s>'
+            '</voice>'
+            '</prosody>'
+            '</prosody>'
+            '</speak>'
+        )
+        with open('_tmp_ssml.txt', 'w') as f:
+            f.write(_ssml)
+
+
+        # ps = subprocess.Popen(f'cat _tmp_ssml.txt | mimic3 --ssml > {reference_wav}', shell=True)
+        # ps.wait()  # using ps to call mimic3 because samples dont have time to be written in stdout buffer
+        args = get_args()
+        args.ssml = True
+        args.text = [_ssml]  #['aa', 'bb'] #txt
+        args.interactive = False
+        # args.output_naming = OutputNaming.TIME
+
+        state = CommandLineInterfaceState(args=args)
+        initialize_args(state)
+        initialize_tts(state)
+        # args.texts = [txt] #['aa', 'bb'] #txt
+        # state.stdout = '.' #None #'makeme.wav'
+        # state.output_dir = '.noopy'
+        # state.interactive = False
+        # state.output_naming = OutputNaming.TIME
+        # # state.ssml = 1234546575
+        # state.stdout = True
+        # state.tts = True
+        process_lines(state, wav_path=reference_wav)
+        shutdown_tts(state)
+        print(os.path.getsize(reference_wav), 'SZ')

harvard.json

@@ -0,0 +1,1158 @@
+{
+    "sentences": [
+      {
+        "list": 1,
+        "sentences": [
+          "The birch canoe slid on the smooth planks.",
+          "Glue the sheet to the dark blue background.",
+          "It's easy to tell the depth of a well.",
+          "These days a chicken leg is a rare dish.",
+          "Rice is often served in round bowls.",
+          "The juice of lemons makes fine punch.",
+          "The box was thrown beside the parked truck.",
+          "The hogs were fed chopped corn and garbage.",
+          "Four hours of steady work faced us.",
+          "Large size in stockings is hard to sell."
+        ]
+      },
+      {
+        "list": 2,
+        "sentences": [
+          "The boy was there when the sun rose.",
+          "A rod is used to catch pink salmon.",
+          "The source of the huge river is the clear spring.",
+          "Kick the ball straight and follow through.",
+          "Help the woman get back to her feet.",
+          "A pot of tea helps to pass the evening.",
+          "Smoky fires lack flame and heat.",
+          "The soft cushion broke the man's fall.",
+          "The salt breeze came across from the sea.",
+          "The girl at the booth sold fifty bonds."
+        ]
+      },
+      {
+        "list": 3,
+        "sentences": [
+          "The small pup gnawed a hole in the sock.",
+          "The fish twisted and turned on the bent hook.",
+          "Press the pants and sew a button on the vest.",
+          "The swan dive was far short of perfect.",
+          "The beauty of the view stunned the young boy.",
+          "Two blue fish swam in the tank.",
+          "Her purse was full of useless trash.",
+          "The colt reared and threw the tall rider.",
+          "It snowed, rained, and hailed the same morning.",
+          "Read verse out loud for pleasure."
+        ]
+      },
+      {
+        "list": 4,
+        "sentences": [
+          "Hoist the load to your left shoulder.",
+          "Take the winding path to reach the lake.",
+          "Note closely the size of the gas tank.",
+          "Wipe the grease off your dirty face.",
+          "Mend the coat before you go out.",
+          "The wrist was badly strained and hung limp.",
+          "The stray cat gave birth to kittens.",
+          "The young girl gave no clear response.",
+          "The meal was cooked before the bell rang.",
+          "What joy there is in living."
+        ]
+      },
+      {
+        "list": 5,
+        "sentences": [
+          "A king ruled the state in the early days.",
+          "The ship was torn apart on the sharp reef.",
+          "Sickness kept him home the third week.",
+          "The wide road shimmered in the hot sun.",
+          "The lazy cow lay in the cool grass.",
+          "Lift the square stone over the fence.",
+          "The rope will bind the seven books at once.",
+          "Hop over the fence and plunge in.",
+          "The friendly gang left the drug store.",
+          "Mesh wire keeps chicks inside."
+        ]
+      },
+      {
+        "list": 6,
+        "sentences": [
+          "The frosty air passed through the coat.",
+          "The crooked maze failed to fool the mouse.",
+          "Adding fast leads to wrong sums.",
+          "The show was a flop from the very start.",
+          "A saw is a tool used for making boards.",
+          "The wagon moved on well-oiled wheels.",
+          "March the soldiers past the next hill.",
+          "A cup of sugar makes sweet fudge.",
+          "Place a rosebush near the porch steps.",
+          "Both lost their lives in the raging storm."
+        ]
+      },
+      {
+        "list": 7,
+        "sentences": [
+          "We talked of the side show in the circus.",
+          "Use a pencil to write the first draft.",
+          "He ran half a mile and then walked.",
+          "The sky that morning was clear and bright blue.",
+          "The tame animal lived in the zoo.",
+          "The dam broke with a loud crash.",
+          "The map helped us find our way.",
+          "He carved a toy from a tiny block of wood.",
+          "The first part of the plan needs changing.",
+          "The new girl was the pride of the class."
+        ]
+      },
+      {
+        "list": 8,
+        "sentences": [
+          "The spear was thrown with great force.",
+          "They took the axe and the saw.",
+          "The ancient coin was quite dull.",
+          "The rush for gold made them rich.",
+          "The noontime sun made the air hot.",
+          "The pine trees were cut down.",
+          "The girl worked hard and with care.",
+          "The church was built of stone.",
+          "The two men played a game of cards.",
+          "The ship sailed across the blue sea."
+        ]
+      },
+      {
+        "list": 9,
+        "sentences": [
+          "The lease ran out in sixteen weeks.",
+          "A font of type is stored in the case.",
+          "We need grain to keep our mules healthy.",
+          "The ink stain dried on the finished page.",
+          "The walled town was seized without a fight.",
+          "The lease will be up in three weeks.",
+          "The purple tie was ten years old.",
+          "Men think and plan and sometimes act.",
+          "A six-shooter is a gun that fires six shots.",
+          "A shoe of fine quality costs more."
+        ]
+      },
+      {
+        "list": 10,
+        "sentences": [
+          "The house was torn down to make room.",
+          "A map shows where the party will meet.",
+          "The ink stain dried on the finished page.",
+          "The bread was buttered on both sides.",
+          "The corner store was robbed last night.",
+          "The store walls were lined with colored tiles.",
+          "The store was out of chocolate soda.",
+          "The bow of the ship made a deep ripple.",
+          "The coat and vest were slightly worn.",
+          "The bill was paid every third week."
+        ]
+      },
+      {
+        "list": 11,
+        "sentences": [
+          "Oak is strong and also gives shade.",
+          "Cats and dogs each hate the other.",
+          "The pipe began to rust while new.",
+          "Open the crate but don't break the glass.",
+          "Add the sum to the product of these three.",
+          "Thieves who rob friends deserve jail.",
+          "The ripe taste of cheese improves with age.",
+          "Act on these orders with great speed.",
+          "The hog crawled under the high fence.",
+          "Move the vat over the hot fire."
+        ]
+      },
+      {
+        "list": 12,
+        "sentences": [
+          "The bark of the pine tree was shiny and dark.",
+          "Leaves turn brown and yellow in the fall.",
+          "The pennant waved when the wind blew.",
+          "Split the log with a quick, sharp blow.",
+          "Burn peat after the logs give out.",
+          "He ordered peach pie with ice cream.",
+          "We saw the march begin on time.",
+          "Tack the strip of carpet to the worn stairs.",
+          "It is easy to tell the depth of a well.",
+          "Four hours of steady work faced us."
+        ]
+      },
+      {
+        "list": 13,
+        "sentences": [
+          "They told us to dip our bread in the gravy.",
+          "Please pack it in the old, green trunk.",
+          "The strange noise drove us from the room.",
+          "We talked of the side show in the circus.",
+          "Use a pencil to write the first draft.",
+          "Three years later, the shell is thin.",
+          "The clock struck to mark the third period.",
+          "A small creek cut across the field.",
+          "Cars and busses stalled in snow drifts.",
+          "The set of china hit the floor with a crash."
+        ]
+      },
+      {
+        "list": 14,
+        "sentences": [
+          "This is a grand season for hikes on the road.",
+          "The dune rose from the edge of the water.",
+          "Those words were the cue for the actor to leave.",
+          "A yacht slid around the point into the bay.",
+          "The two met while playing on the sand.",
+          "The ink stain dried on the finished page.",
+          "The walled town was seized without a fight.",
+          "The lease will be up in three weeks.",
+          "The purple tie was ten years old.",
+          "Men think and plan and sometimes act."
+        ]
+      },
+      {
+        "list": 15,
+        "sentences": [
+          "She has been ill, and will go home soon.",
+          "The weasel will be trapped on a ledge.",
+          "The beak of the bird is very sharp.",
+          "The boy was there when the sun rose.",
+          "A rod is used to catch pink salmon.",
+          "The source of the huge river is the clear spring.",
+          "Kick the ball straight and follow through.",
+          "Help the woman get back to her feet.",
+          "A pot of tea helps to pass the evening.",
+          "Smoky fires lack flame and heat."
+        ]
+      },
+      {
+        "list": 16,
+        "sentences": [
+          "The soft cushion broke the man's fall.",
+          "The salt breeze came across from the sea.",
+          "The girl at the booth sold fifty bonds.",
+          "The small pup gnawed a hole in the sock.",
+          "The fish twisted and turned on the bent hook.",
+          "Press the pants and sew a button on the vest.",
+          "The swan dive was far short of perfect.",
+          "The beauty of the view stunned the young boy.",
+          "Two blue fish swam in the tank.",
+          "Her purse was full of useless trash."
+        ]
+      },
+      {
+        "list": 17,
+        "sentences": [
+          "The colt reared and threw the tall rider.",
+          "It snowed, rained, and hailed the same morning.",
+          "Read verse out loud for pleasure.",
+          "Hoist the load to your left shoulder.",
+          "Take the winding path to reach the lake.",
+          "Note closely the size of the gas tank.",
+          "Wipe the grease off your dirty face.",
+          "Mend the coat before you go out.",
+          "The wrist was badly strained and hung limp.",
+          "The stray cat gave birth to kittens."
+        ]
+      },
+      {
+        "list": 18,
+        "sentences": [
+          "The young girl gave no clear response.",
+          "The meal was cooked before the bell rang.",
+          "What joy there is in living.",
+          "A king ruled the state in the early days.",
+          "The ship was torn apart on the sharp reef.",
+          "Sickness kept him home the third week.",
+          "The wide road shimmered in the hot sun.",
+          "The lazy cow lay in the cool grass.",
+          "Lift the square stone over the fence.",
+          "The rope will bind the seven books at once."
+        ]
+      },
+      {
+        "list": 19,
+        "sentences": [
+          "Hop over the fence and plunge in.",
+          "The friendly gang left the drug store.",
+          "Mesh wire keeps chicks inside.",
+          "The frosty air passed through the coat.",
+          "The crooked maze failed to fool the mouse.",
+          "Adding fast leads to wrong sums.",
+          "The show was a flop from the very start.",
+          "A saw is a tool used for making boards.",
+          "The wagon moved on well-oiled wheels.",
+          "March the soldiers past the next hill."
+        ]
+      },
+      {
+        "list": 20,
+        "sentences": [
+          "A cup of sugar makes sweet fudge.",
+          "Place a rosebush near the porch steps.",
+          "Both lost their lives in the raging storm.",
+          "We talked of the side show in the circus.",
+          "Use a pencil to write the first draft.",
+          "He ran half a mile and then walked.",
+          "The sky that morning was clear and bright blue.",
+          "The tame animal lived in the zoo.",
+          "The dam broke with a loud crash.",
+          "The map helped us find our way."
+        ]
+      },
+      {
+        "list": 21,
+        "sentences": [
+          "He carved a toy from a tiny block of wood.",
+          "The first part of the plan needs changing.",
+          "The new girl was the pride of the class.",
+          "The spear was thrown with great force.",
+          "They took the axe and the saw.",
+          "The ancient coin was quite dull.",
+          "The rush for gold made them rich.",
+          "The noontime sun made the air hot.",
+          "The pine trees were cut down.",
+          "The girl worked hard and with care."
+        ]
+      },
+      {
+        "list": 22,
+        "sentences": [
+          "The church was built of stone.",
+          "The two men played a game of cards.",
+          "The ship sailed across the blue sea.",
+          "The lease ran out in sixteen weeks.",
+          "A font of type is stored in the case.",
+          "We need grain to keep our mules healthy.",
+          "The ink stain dried on the finished page.",
+          "The walled town was seized without a fight.",
+          "The lease will be up in three weeks.",
+          "The purple tie was ten years old."
+        ]
+      },
+      {
+        "list": 23,
+        "sentences": [
+          "Men think and plan and sometimes act.",
+          "A six-shooter is a gun that fires six shots.",
+          "A shoe of fine quality costs more.",
+          "The house was torn down to make room.",
+          "A map shows where the party will meet.",
+          "The ink stain dried on the finished page.",
+          "The bread was buttered on both sides.",
+          "The corner store was robbed last night.",
+          "The store walls were lined with colored tiles.",
+          "The store was out of chocolate soda."
+        ]
+      },
+      {
+        "list": 24,
+        "sentences": [
+          "The bow of the ship made a deep ripple.",
+          "The coat and vest were slightly worn.",
+          "The bill was paid every third week.",
+          "Clear water over gravel shone like silver.",
+          "The wolf came from the cold, dark forest.",
+          "Coax a young calf to drink from a bucket.",
+          "Schools for ladies teach charm and grace.",
+          "The lamp shone with a steady green flame.",
+          "They took the axe and the saw.",
+          "The ancient coin was quite dull."
+        ]
+      },
+      {
+        "list": 25,
+        "sentences": [
+          "The rush for gold made them rich.",
+          "The noontime sun made the air hot.",
+          "The pine trees were cut down.",
+          "The girl worked hard and with care.",
+          "The church was built of stone.",
+          "The two men played a game of cards.",
+          "The ship sailed across the blue sea.",
+          "The lease ran out in sixteen weeks.",
+          "A font of type is stored in the case.",
+          "We need grain to keep our mules healthy."
+        ]
+      },
+      {
+        "list": 26,
+        "sentences": [
+          "The ink stain dried on the finished page.",
+          "The walled town was seized without a fight.",
+          "The lease will be up in three weeks.",
+          "The purple tie was ten years old.",
+          "Men think and plan and sometimes act.",
+          "A six-shooter is a gun that fires six shots.",
+          "A shoe of fine quality costs more.",
+          "The house was torn down to make room.",
+          "A map shows where the party will meet.",
+          "The ink stain dried on the finished page."
+        ]
+      },
+      {
+        "list": 27,
+        "sentences": [
+          "The bread was buttered on both sides.",
+          "The corner store was robbed last night.",
+          "The store walls were lined with colored tiles.",
+          "The store was out of chocolate soda.",
+          "The bow of the ship made a deep ripple.",
+          "The coat and vest were slightly worn.",
+          "The bill was paid every third week.",
+          "Clear water over gravel shone like silver.",
+          "The wolf came from the cold, dark forest.",
+          "Coax a young calf to drink from a bucket."
+        ]
+      },
+      {
+        "list": 28,
+        "sentences": [
+          "Schools for ladies teach charm and grace.",
+          "The lamp shone with a steady green flame.",
+          "It was done before the boy could see it.",
+          "March the soldiers past the next hill.",
+          "A cup of sugar makes sweet fudge.",
+          "Place a rosebush near the porch steps.",
+          "Both lost their lives in the raging storm.",
+          "We talked of the side show in the circus.",
+          "Use a pencil to write the first draft.",
+          "He ran half a mile and then walked."
+        ]
+      },
+      {
+        "list": 29,
+        "sentences": [
+          "The sky that morning was clear and bright blue.",
+          "The tame animal lived in the zoo.",
+          "The dam broke with a loud crash.",
+          "The map helped us find our way.",
+          "He carved a toy from a tiny block of wood.",
+          "The first part of the plan needs changing.",
+          "The new girl was the pride of the class.",
+          "The spear was thrown with great force.",
+          "They took the axe and the saw.",
+          "The ancient coin was quite dull."
+        ]
+      },
+      {
+        "list": 30,
+        "sentences": [
+          "The rush for gold made them rich.",
+          "The noontime sun made the air hot.",
+          "The pine trees were cut down.",
+          "The girl worked hard and with care.",
+          "The church was built of stone.",
+          "The two men played a game of cards.",
+          "The ship sailed across the blue sea.",
+          "The lease ran out in sixteen weeks.",
+          "A font of type is stored in the case.",
+          "We need grain to keep our mules healthy."
+        ]
+      },
+      {
+        "list": 31,
+        "sentences": [
+          "The ink stain dried on the finished page.",
+          "The walled town was seized without a fight.",
+          "The lease will be up in three weeks.",
+          "The purple tie was ten years old.",
+          "Men think and plan and sometimes act.",
+          "A six-shooter is a gun that fires six shots.",
+          "A shoe of fine quality costs more.",
+          "The house was torn down to make room.",
+          "A map shows where the party will meet.",
+          "The ink stain dried on the finished page."
+        ]
+      },
+      {
+        "list": 32,
+        "sentences": [
+          "The bread was buttered on both sides.",
+          "The corner store was robbed last night.",
+          "The store walls were lined with colored tiles.",
+          "The store was out of chocolate soda.",
+          "The bow of the ship made a deep ripple.",
+          "The coat and vest were slightly worn.",
+          "The bill was paid every third week.",
+          "Clear water over gravel shone like silver.",
+          "The wolf came from the cold, dark forest.",
+          "Coax a young calf to drink from a bucket."
+        ]
+      },
+      {
+        "list": 33,
+        "sentences": [
+          "Schools for ladies teach charm and grace.",
+          "The lamp shone with a steady green flame.",
+          "It was done before the boy could see it.",
+          "March the soldiers past the next hill.",
+          "A cup of sugar makes sweet fudge.",
+          "Place a rosebush near the porch steps.",
+          "Both lost their lives in the raging storm.",
+          "We talked of the side show in the circus.",
+          "Use a pencil to write the first draft.",
+          "He ran half a mile and then walked."
+        ]
+      },
+      {
+        "list": 34,
+        "sentences": [
+          "The sky that morning was clear and bright blue.",
+          "The tame animal lived in the zoo.",
+          "The dam broke with a loud crash.",
+          "The map helped us find our way.",
+          "He carved a toy from a tiny block of wood.",
+          "The first part of the plan needs changing.",
+          "The new girl was the pride of the class.",
+          "The spear was thrown with great force.",
+          "They took the axe and the saw.",
+          "The ancient coin was quite dull."
+        ]
+      },
+      {
+        "list": 35,
+        "sentences": [
+          "The rush for gold made them rich.",
+          "The noontime sun made the air hot.",
+          "The pine trees were cut down.",
+          "The girl worked hard and with care.",
+          "The church was built of stone.",
+          "The two men played a game of cards.",
+          "The ship sailed across the blue sea.",
+          "The lease ran out in sixteen weeks.",
+          "A font of type is stored in the case.",
+          "We need grain to keep our mules healthy."
+        ]
+      },
+      {
+        "list": 36,
+        "sentences": [
+          "The ink stain dried on the finished page.",
+          "The walled town was seized without a fight.",
+          "The lease will be up in three weeks.",
+          "The purple tie was ten years old.",
+          "Men think and plan and sometimes act.",
+          "A six-shooter is a gun that fires six shots.",
+          "A shoe of fine quality costs more.",
+          "The house was unhitherto."
+        ]
+      },
+      {
+      "list": 37,
+      "sentences": [
+        "The bread was buttered on both sides.",
+        "The corner store was robbed last night.",
+        "The store walls were lined with colored tiles.",
+        "The store was out of chocolate soda.",
+        "The bow of the ship made a deep ripple.",
+        "The coat and vest were slightly worn.",
+        "The bill was paid every third week.",
+        "Clear water over gravel shone like silver.",
+        "The wolf came from the cold, dark forest.",
+        "Coax a young calf to drink from a bucket."
+      ]
+    },
+    {
+      "list": 38,
+      "sentences": [
+        "Schools for ladies teach charm and grace.",
+        "The lamp shone with a steady green flame.",
+        "It was done before the boy could see it.",
+        "March the soldiers past the next hill.",
+        "A cup of sugar makes sweet fudge.",
+        "Place a rosebush near the porch steps.",
+        "Both lost their lives in the raging storm.",
+        "We talked of the side show in the circus.",
+        "Use a pencil to write the first draft.",
+        "He ran half a mile and then walked."
+      ]
+    },
+    {
+      "list": 39,
+      "sentences": [
+        "The sky that morning was clear and bright blue.",
+        "The tame animal lived in the zoo.",
+        "The dam broke with a loud crash.",
+        "The map helped us find our way.",
+        "He carved a toy from a tiny block of wood.",
+        "The first part of the plan needs changing.",
+        "The new girl was the pride of the class.",
+        "The spear was thrown with great force.",
+        "They took the axe and the saw.",
+        "The ancient coin was quite dull."
+      ]
+    },
+    {
+      "list": 40,
+      "sentences": [
+        "The rush for gold made them rich.",
+        "The noontime sun made the air hot.",
+        "The pine trees were cut down.",
+        "The girl worked hard and with care.",
+        "The church was built of stone.",
+        "The two men played a game of cards.",
+        "The ship sailed across the blue sea.",
+        "The lease ran out in sixteen weeks.",
+        "A font of type is stored in the case.",
+        "We need grain to keep our mules healthy."
+      ]
+    },
+    {
+      "list": 41,
+      "sentences": [
+        "The ink stain dried on the finished page.",
+        "The walled town was seized without a fight.",
+        "The lease will be up in three weeks.",
+        "The purple tie was ten years old.",
+        "Men think and plan and sometimes act.",
+        "A six-shooter is a gun that fires six shots.",
+        "A shoe of fine quality costs more.",
+        "The house was torn down to make room.",
+        "A map shows where the party will meet.",
+        "The ink stain dried on the finished page."
+      ]
+    },
+    {
+      "list": 42,
+      "sentences": [
+        "The bread was buttered on both sides.",
+        "The corner store was robbed last night.",
+        "The store walls were lined with colored tiles.",
+        "The store was out of chocolate soda.",
+        "The bow of the ship made a deep ripple.",
+        "The coat and vest were slightly worn.",
+        "The bill was paid every third week.",
+        "Clear water over gravel shone like silver.",
+        "The wolf came from the cold, dark forest.",
+        "Coax a young calf to drink from a bucket."
+      ]
+    },
+    {
+      "list": 43,
+      "sentences": [
+        "Schools for ladies teach charm and grace.",
+        "The lamp shone with a steady green flame.",
+        "It was done before the boy could see it.",
+        "March the soldiers past the next hill.",
+        "A cup of sugar makes sweet fudge.",
+        "Place a rosebush near the porch steps.",
+        "Both lost their lives in the raging storm.",
+        "We talked of the side show in the circus.",
+        "Use a pencil to write the first draft.",
+        "He ran half a mile and then walked."
+      ]
+    },
+    {
+      "list": 44,
+      "sentences": [
+        "The sky that morning was clear and bright blue.",
+        "The tame animal lived in the zoo.",
+        "The dam broke with a loud crash.",
+        "The map helped us find our way.",
+        "He carved a toy from a tiny block of wood.",
+        "The first part of the plan needs changing.",
+        "The new girl was the pride of the class.",
+        "The spear was thrown with great force.",
+        "They took the axe and the saw.",
+        "The ancient coin was quite dull."
+      ]
+    },
+    {
+      "list": 45,
+      "sentences": [
+        "The rush for gold made them rich.",
+        "The noontime sun made the air hot.",
+        "The pine trees were cut down.",
+        "The girl worked hard and with care.",
+        "The church was built of stone.",
+        "The two men played a game of cards.",
+        "The ship sailed across the blue sea.",
+        "The lease ran out in sixteen weeks.",
+        "A font of type is stored in the case.",
+        "We need grain to keep our mules healthy."
+      ]
+    },
+    {
+      "list": 46,
+      "sentences": [
+        "The ink stain dried on the finished page.",
+        "The walled town was seized without a fight.",
+        "The lease will be up in three weeks.",
+        "The purple tie was ten years old.",
+        "Men think and plan and sometimes act.",
+        "A six-shooter is a gun that fires six shots.",
+        "A shoe of fine quality costs more.",
+        "The house was torn down to make room.",
+        "A map shows where the party will meet.",
+        "The ink stain dried on the finished page."
+      ]
+    },
+    {
+      "list": 47,
+      "sentences": [
+        "The bread was buttered on both sides.",
+        "The corner store was robbed last night.",
+        "The store walls were lined with colored tiles.",
+        "The store was out of chocolate soda.",
+        "The bow of the ship made a deep ripple.",
+        "The coat and vest were slightly worn.",
+        "The bill was paid every third week.",
+        "Clear water over gravel shone like silver.",
+        "The wolf came from the cold, dark forest.",
+        "Coax a young calf to drink from a bucket."
+      ]
+    },
+    {
+      "list": 48,
+      "sentences": [
+        "Schools for ladies teach charm and grace.",
+        "The lamp shone with a steady green flame.",
+        "It was done before the boy could see it.",
+        "March the soldiers past the next hill.",
+        "A cup of sugar makes sweet fudge.",
+        "Place a rosebush near the porch steps.",
+        "Both lost their lives in the raging storm.",
+        "We talked of the side show in the circus.",
+        "Use a pencil to write the first draft.",
+        "He ran half a mile and then walked."
+      ]
+    },
+    {
+      "list": 49,
+      "sentences": [
+        "The sky that morning was clear and bright blue.",
+        "The tame animal lived in the zoo.",
+        "The dam broke with a loud crash.",
+        "The map helped us find our way.",
+        "He carved a toy from a tiny block of wood.",
+        "The first part of the plan needs changing.",
+        "The new girl was the pride of the class.",
+        "The spear was thrown with great force.",
+        "They took the axe and the saw.",
+        "The ancient coin was quite dull."
+      ]
+    },
+    {
+      "list": 50,
+      "sentences": [
+        "The rush for gold made them rich.",
+        "The noontime sun made the air hot.",
+        "The pine trees were cut down.",
+        "The girl worked hard and with care.",
+        "The church was built of stone.",
+        "The two men played a game of cards.",
+        "The ship sailed across the blue sea.",
+        "The lease ran out in sixteen weeks.",
+        "A font of type is stored in the case.",
+        "We need grain to keep our mules healthy."
+      ]
+    },
+    {
+          "list": 51,
+          "sentences": [
+            "The chalk was used to mark the wall.",
+            "The clock ticked steadily in the quiet room.",
+            "She made a cup of tea with lemon.",
+            "The dog barked loudly at the passing cars.",
+            "The bright sun warmed the cool air.",
+            "The children played in the sunny park.",
+            "He fixed the leak under the sink.",
+            "The book was filled with interesting facts.",
+            "The phone rang incessantly during dinner.",
+            "They walked through the beautiful garden."
+          ]
+        },
+        {
+          "list": 52,
+          "sentences": [
+            "The car broke down on the busy highway.",
+            "The cat sat lazily by the window.",
+            "The wind rustled the leaves of the trees.",
+            "She wore a bright red dress to the party.",
+            "The painting hung prominently on the wall.",
+            "The coffee was too hot to drink.",
+            "He read the newspaper over breakfast.",
+            "The house was decorated for the holidays.",
+            "The kids enjoyed their time at the amusement park.",
+            "She bought a new pair of shoes for the event."
+          ]
+        },
+        {
+          "list": 53,
+          "sentences": [
+            "The road was blocked due to construction.",
+            "The lamp on the table cast a warm glow.",
+            "He wrote a letter to his old friend.",
+            "The bicycle was parked outside the house.",
+            "The meeting was scheduled for the afternoon.",
+            "She filled the vase with fresh flowers.",
+            "The wind chimes tinkled softly in the breeze.",
+            "They went hiking in the mountains over the weekend.",
+            "The cookies were freshly baked and smelled delicious.",
+            "The new movie was a big hit at the box office."
+          ]
+        },
+        {
+          "list": 54,
+          "sentences": [
+            "The boat floated gently on the calm water.",
+            "She enjoyed reading a good book in the evening.",
+            "The students prepared for their final exams.",
+            "The restaurant served a variety of cuisines.",
+            "The picture frame was made of polished wood.",
+            "The conference was held in a large hall.",
+            "He repaired the old wooden fence.",
+            "The bakery sold a selection of fresh bread.",
+            "They watched the sunset from the beach.",
+            "The garden was full of colorful flowers."
+          ]
+        },
+        {
+          "list": 55,
+          "sentences": [
+            "The train arrived at the station on time.",
+            "She organized her desk before starting work.",
+            "The guitar was hung on the wall as decoration.",
+            "The chocolate cake was rich and creamy.",
+            "He planned a surprise party for his friend.",
+            "The weather forecast predicted rain for the weekend.",
+            "The children were excited for the field trip.",
+            "She made a list of groceries to buy.",
+            "The old book had a musty smell.",
+            "They enjoyed a quiet dinner at home."
+          ]
+        },
+        {
+          "list": 56,
+          "sentences": [
+            "The movie was entertaining and well-received.",
+            "She carefully folded the letter and put it in an envelope.",
+            "The cat played with a ball of yarn.",
+            "The old bridge spanned the wide river.",
+            "He wore a suit to the formal event.",
+            "The library had a vast collection of books.",
+            "The flowers in the garden were in full bloom.",
+            "She made a delicious stew for dinner.",
+            "The car’s engine roared to life.",
+            "They sat around the campfire telling stories."
+          ]
+        },
+        {
+          "list": 57,
+          "sentences": [
+            "The painting was framed and hung on the wall.",
+            "She checked her email for new messages.",
+            "The old clock struck twelve.",
+            "He fixed the broken window pane.",
+            "The park was a great place for a picnic.",
+            "The bakery had a special on pastries.",
+            "She enjoyed a relaxing bath in the evening.",
+            "The book was a bestseller.",
+            "They traveled to a new city for vacation.",
+            "The kids played on the swings and slides."
+          ]
+        },
+        {
+          "list": 58,
+          "sentences": [
+            "The store had a sale on winter coats.",
+            "She made a reservation for dinner at the restaurant.",
+            "The dog chased after the ball in the yard.",
+            "He read the instructions carefully before starting.",
+            "The house had a large backyard.",
+            "They went to the theater to see a play.",
+            "The coffee shop had a cozy atmosphere.",
+            "She planted tomatoes in her garden.",
+            "The car needed a new set of tires.",
+            "The museum exhibited ancient artifacts."
+          ]
+        },
+        {
+          "list": 59,
+          "sentences": [
+            "The sun set behind the mountains.",
+            "She wrapped the gift in colorful paper.",
+            "The kids enjoyed a fun day at the zoo.",
+            "The phone’s battery was running low.",
+            "He fixed the leaking faucet in the kitchen.",
+            "The concert was held in a large auditorium.",
+            "She attended a workshop on cooking.",
+            "The laptop was used for work and study.",
+            "They went skiing in the winter.",
+            "The cat napped on the sunny windowsill."
+          ]
+        },
+        {
+          "list": 60,
+          "sentences": [
+            "The fireworks lit up the night sky.",
+            "She wore a hat to protect herself from the sun.",
+            "The book was placed on the top shelf.",
+            "He enjoyed a cup of hot cocoa by the fireplace.",
+            "The children built a sandcastle at the beach.",
+            "The store offered a discount on electronics.",
+            "She found a new recipe for apple pie.",
+            "The dog wagged its tail happily.",
+            "They explored a cave during their hike.",
+            "The painting was a gift from a friend."
+          ]
+        },
+        {
+          "list": 61,
+          "sentences": [
+            "The flowers in the vase were wilted.",
+            "She organized her closet and donated old clothes.",
+            "The children played board games indoors.",
+            "The coffee was brewed fresh every morning.",
+            "He visited a historical site during his trip.",
+            "The car’s air conditioning was broken.",
+            "She attended a seminar on personal finance.",
+            "The laptop screen was cracked.",
+            "They enjoyed a quiet evening at home.",
+            "The library had a section for new arrivals."
+          ]
+        },
+        {
+          "list": 62,
+          "sentences": [
+            "The sunset painted the sky with shades of orange.",
+            "She added a pinch of salt to the soup.",
+            "The old house had a charming front porch.",
+            "He finished his homework before dinner.",
+            "The store had a wide selection of books.",
+            "She planted flowers in the spring.",
+            "The dog loved going for walks in the park.",
+            "The new movie was a box office success.",
+            "They enjoyed a barbecue on the weekend.",
+            "The clock on the wall ticked softly."
+          ]
+        },
+        {
+          "list": 63,
+          "sentences": [
+            "The ice cream melted quickly in the heat.",
+            "She arranged the books neatly on the shelf.",
+            "The car was parked in the garage.",
+            "He wrote a poem for his friend's birthday.",
+            "The garden was full of blooming roses.",
+            "She prepared a special dinner for the occasion.",
+            "The kids watched a movie on the big screen.",
+            "The coffee shop had a variety of pastries.",
+            "The hotel room had a beautiful view.",
+            "They took a scenic drive through the countryside."
+          ]
+        },
+        {
+          "list": 64,
+          "sentences": [
+            "The rain pattered gently on the roof.",
+            "She set the table with fine china.",
+            "The book was about a thrilling adventure.",
+            "He cleaned the house from top to bottom.",
+            "The mountain trail was steep and rocky.",
+            "She enjoyed reading a novel in the evening.",
+            "The kids played with their new toys.",
+            "The museum had an exhibit on ancient Egypt.",
+            "He fixed the broken chair with new screws.",
+            "The park was filled with the sounds of children."
+          ]
+        },
+        {
+          "list": 65,
+          "sentences": [
+            "The concert tickets sold out quickly.",
+            "She found a quaint little café for lunch.",
+            "The plant needed more sunlight to grow.",
+            "The movie had an unexpected plot twist.",
+            "He repaired the old wooden fence in the backyard.",
+            "The restaurant was known for its seafood.",
+            "She painted the room a calming blue.",
+            "The cat slept peacefully on the couch.",
+            "They went to a vineyard for a wine tasting.",
+            "The new phone had advanced features."
+          ]
+        },
+        {
+          "list": 66,
+          "sentences": [
+            "The summer breeze was refreshing.",
+            "She made a scrapbook of her travels.",
+            "The dog enjoyed playing fetch in the yard.",
+            "The house had a spacious living room.",
+            "He wrote a thank-you note for the gift.",
+            "The vacation was a relaxing escape.",
+            "She enjoyed a cup of herbal tea.",
+            "The car had a spacious trunk.",
+            "They visited a botanical garden.",
+            "The painting was vibrant and colorful."
+          ]
+        },
+        {
+              "list": 67,
+              "sentences": [
+                "The book was left on the kitchen counter.",
+                "She carefully folded the letter and put it in an envelope.",
+                "The children played with their new toys.",
+                "He made a cup of coffee to start the day.",
+                "The new restaurant received great reviews.",
+                "The old bridge was recently renovated.",
+                "She decorated the room with fresh flowers.",
+                "The dog slept soundly in its bed.",
+                "They planned a trip to the mountains.",
+                "The phone charger was misplaced."
+              ]
+            },
+            {
+              "list": 68,
+              "sentences": [
+                "The cake was decorated with colorful frosting.",
+                "She found a charming café on her walk.",
+                "The car was washed and vacuumed.",
+                "He repaired the fence in the backyard.",
+                "The movie had a surprising ending.",
+                "The garden was filled with blooming tulips.",
+                "She wore a hat to shield herself from the sun.",
+                "The old clock on the wall ticked loudly.",
+                "They went hiking in the nearby hills.",
+                "The museum had an exhibit on modern art."
+              ]
+            },
+            {
+              "list": 69,
+              "sentences": [
+                "The weather was perfect for a picnic.",
+                "She baked a fresh loaf of bread.",
+                "The dog ran happily through the park.",
+                "He finished reading the novel last night.",
+                "The house was decorated for the holidays.",
+                "They enjoyed a leisurely breakfast.",
+                "The car broke down on the way home.",
+                "She wore a scarf to keep warm.",
+                "The library had a quiet reading room.",
+                "The restaurant had a cozy atmosphere."
+              ]
+            },
+            {
+              "list": 70,
+              "sentences": [
+                "The sunset cast a beautiful orange glow.",
+                "She made a fresh pot of tea.",
+                "The children played in the backyard.",
+                "He organized his books by genre.",
+                "The hotel room had a stunning view.",
+                "The new album was a big hit.",
+                "She found a lovely vase at the market.",
+                "The dog barked at the mail carrier.",
+                "They visited a vineyard for a tour.",
+                "The house had a charming front porch."
+              ]
+            },
+            {
+              "list": 71,
+              "sentences": [
+                "The old painting was restored to its former glory.",
+                "She enjoyed a relaxing evening at home.",
+                "The store had a sale on winter apparel.",
+                "The children made a fort out of blankets.",
+                "He fixed the leaky faucet in the kitchen.",
+                "The concert was held in an outdoor arena.",
+                "She bought a new book to read.",
+                "The garden was lush and green.",
+                "They went to the beach for a swim.",
+                "The cat napped on a sunny windowsill."
+              ]
+            },
+            {
+              "list": 72,
+              "sentences": [
+                "The old oak tree provided ample shade.",
+                "She made a delicious pie from scratch.",
+                "The new phone had a high-resolution camera.",
+                "The dog enjoyed chasing the ball.",
+                "He attended a workshop on photography.",
+                "The library had a wide selection of magazines.",
+                "The kids played with their new board game.",
+                "The painting was a gift from a friend.",
+                "She wore a new dress to the party.",
+                "The car's engine roared to life."
+              ]
+            },
+            {
+              "list": 73,
+              "sentences": [
+                "The cake was decorated with intricate designs.",
+                "She enjoyed a warm cup of cocoa by the fire.",
+                "The garden was full of blooming roses.",
+                "He cleaned the house from top to bottom.",
+                "The movie had a captivating storyline.",
+                "The old house had a beautiful staircase.",
+                "She wrote a letter to her grandmother.",
+                "The dog loved playing in the backyard.",
+                "They had a barbecue in the garden.",
+                "The museum had an interesting collection."
+              ]
+            },
+            {
+              "list": 74,
+              "sentences": [
+                "The summer heat made everyone uncomfortable.",
+                "She made a reservation for dinner at a new restaurant.",
+                "The children played outside in the warm sun.",
+                "The car had a comfortable interior.",
+                "He read a thrilling mystery novel.",
+                "The park was filled with the sounds of laughter.",
+                "The house was decorated for the festive season.",
+                "She baked a batch of cookies for her friends.",
+                "The new sofa was very comfortable.",
+                "They visited a local art gallery."
+              ]
+            },
+            {
+              "list": 75,
+              "sentences": [
+                "The fresh flowers brightened up the room.",
+                "She enjoyed a scenic drive through the countryside.",
+                "The children were excited for their field trip.",
+                "He organized a surprise party for his friend.",
+                "The old barn was renovated into a cozy home.",
+                "The cake was frosted with chocolate icing.",
+                "The weather was perfect for a day at the park.",
+                "She found a new hobby in painting.",
+                "The car had a sleek design.",
+                "They spent the afternoon exploring the city."
+              ]
+            },
+            {
+              "list": 76,
+              "sentences": [
+                "The book was well-written and engaging.",
+                "She enjoyed a peaceful evening at home.",
+                "The restaurant had a diverse menu.",
+                "The children played games in the backyard.",
+                "He fixed the broken windowpane.",
+                "The garden was full of vibrant flowers.",
+                "She made a delicious salad for lunch.",
+                "The movie was both entertaining and thought-provoking.",
+                "They visited a historical landmark.",
+                "The new shoes were stylish and comfortable."
+              ]
+            },
+            {
+              "list": 77,
+              "sentences": [
+                "The sun set behind the hills.",
+                "She made a fresh batch of lemonade.",
+                "The children made a snowman in the yard.",
+                "He repaired the leaky roof.",
+                "The park had a beautiful walking trail.",
+                "She enjoyed a warm blanket by the fireplace.",
+                "The book was a thrilling read.",
+                "They went to a festival in the town square.",
+                "The new kitchen appliances were installed.",
+                "The cat slept peacefully on the windowsill."
+              ]
+            }     
+]    
+}
+  

tts_harvard.py

@@ -0,0 +1,142 @@
+# Synthesize all Harvard Lists 77x lists of 10x sentences to single .wav ----- NEEDS TO BE RUN from https://github.com/audeering/shift/
+#
+# 1. using mimic3 style
+#     Folder: 'prompt_mimic3/'
+# 2. using mimic3 4x accelerated style
+#     Folder: 'prompt_mimic3speed/'
+# 3. using crema-d style
+#     Folder: 'prompt_human/'
+#
+# WAVS used from  tts_paper_plot.py
+
+import soundfile
+import json
+import numpy as np
+import audb
+from pathlib import Path
+
+LABELS = ['arousal', 'dominance', 'valence']
+
+
+
+
+def load_speech(split=None):
+    DB = [
+        # [dataset, version, table, has_timdeltas_or_is_full_wavfile]
+           ['crema-d', '1.1.1', 'emotion.voice.test', False],
+ #           ['emodb',  '1.2.0', 'emotion.categories.train.gold_standard', False],
+  #          ['entertain-playtestcloud', '1.1.0', 'emotion.categories.train.gold_standard', True],
+   #         ['erik', '2.2.0', 'emotion.categories.train.gold_standard', True],
+    #        ['meld', '1.3.1', 'emotion.categories.train.gold_standard', False],
+            # ['msppodcast', '5.0.0', 'emotion.categories.train.gold_standard', False],  # tandalone bucket because it has gt labels?
+     #       ['myai', '1.0.1', 'emotion.categories.train.gold_standard', False],
+      #      ['casia', None, 'emotion.categories.gold_standard', False],
+            # ['switchboard-1', None, 'sentiment', True],
+            # ['swiss-parliament', None, 'segments', True], 
+            # ['argentinian-parliament', None, 'segments', True],
+            # ['austrian-parliament', None, 'segments', True],
+            # #'german', --> bundestag
+            # ['brazilian-parliament', None, 'segments', True],
+            # ['mexican-parliament', None, 'segments', True],
+            # ['portuguese-parliament', None, 'segments', True],
+       #     ['spanish-parliament', None, 'segments', True],
+        #    ['chinese-vocal-emotions-liu-pell', None, 'emotion.categories.desired', False],
+            # peoples-speech slow
+         #   ['peoples-speech', None, 'train-initial', False]
+    ]
+
+    output_list = []
+    for database_name, ver, table, has_timedeltas in DB:
+
+        a = audb.load(database_name,
+                        sampling_rate=16000,
+                        format='wav',
+                        mixdown=True,
+                        version=ver,
+                        cache_root='/cache/audb/')
+        a = a[table].get()
+        if has_timedeltas:
+            print(f'{has_timedeltas=}')
+            # a = a.reset_index()[['file', 'start', 'end']]
+            # output_list += [[*t] for t
+            #         in zip(a.file.values, a.start.dt.total_seconds().values, a.end.dt.total_seconds().values)]
+        else:
+            output_list += [f for f in a.index]  # use file (no timedeltas)
+    return output_list
+
+
+
+
+
+# Generate 77 wavs
+
+
+
+
+with open('voices.json', 'r') as f:
+    df = json.load(f)['voices']
+voice_names = [v['voice'] for k,v in df.items()]
+synthetic_wav_paths = []
+synthetic_wav_paths_AFFECT = []
+for voice in voice_names:
+    
+    synthetic_wav_paths.append(
+      'assets/wavs/style_vector/' + voice.replace('/', '_').replace('#', '_').replace(
+                    'cmu-arctic', 'cmu_arctic').replace('_low', '') + '.wav')
+    synthetic_wav_paths_AFFECT.append(
+        'assets/wavs/style_vector_v2/' + voice.replace('/', '_').replace('#', '_').replace(
+                    'cmu-arctic', 'cmu_arctic').replace('_low', '') + '.wav')
+
+
+print(len(synthetic_wav_paths))
+
+    
+natural_wav_paths = load_speech()
+
+
+# SYNTHESIZE mimic mimicx4 crema-d
+import msinference
+
+
+with open('harvard.json', 'r') as f:
+    harvard_individual_sentences = json.load(f)['sentences']
+
+
+
+
+
+for audio_prompt in ['mimic3', 'mimic3_speed', 'human']:
+    total_audio = []
+    ix = 0
+    for list_of_10 in harvard_individual_sentences:
+        # long_sentence = ' '.join(list_of_10['sentences'])
+        # harvard.append(long_sentence.replace('.', ' '))
+        for text in list_of_10['sentences']:
+            if audio_prompt == 'mimic3':
+                style_vec = msinference.compute_style(
+                    synthetic_wav_paths[ix % 134])
+            elif audio_prompt == 'mimic3_speed':
+                style_vec = msinference.compute_style(
+                    synthetic_wav_paths_AFFECT[ix % 134])
+            elif audio_prompt == 'human':
+                style_vec = msinference.compute_style(
+                    natural_wav_paths[ix % len(natural_wav_paths)])
+            else:
+                print('unknonw list of style vecto')
+            print(ix, text)
+            ix += 1
+            x = msinference.inference(text,
+                                        style_vec,
+                                        alpha=0.3,
+                                        beta=0.7,
+                                        diffusion_steps=7,
+                                        embedding_scale=1)
+            
+            total_audio.append(x)
+            # concat before write
+        # -- for 10x sentenctes
+        print('_____________________')
+    # -- for 77x lists
+    total_audio = np.concatenate(total_audio)    
+    soundfile.write(f'{audio_prompt}_770.wav', total_audio, 24000)
+    print(f'{audio_prompt}_full_770.wav')

visualize_tts_plesantness.py

@@ -0,0 +1,371 @@
+# 1. engineer_style_foreign_style_vectors.py  # for speed=1 & speed=4
+# 2. tts_harvard.py                           # (call inside SHIFT repo - needs StyleTTS msinference.py)
+# 3. visualize_tts_pleasantness.py            # figures & audinterface
+
+
+# Visualises timeseries 11 class for mimic3 human mimic3speed
+#
+#
+# human_770.wav
+# mimic3_770.wav
+# mimic3_speedup_770.wav
+import pandas as pd
+import os
+
+import json
+import numpy as np
+import audonnx
+import audb
+from pathlib import Path
+import transformers
+import torch
+import audmodel
+import audinterface
+import matplotlib.pyplot as plt
+import audiofile
+
+LABELS = ['arousal', 'dominance', 'valence',
+           'speech_synthesizer', 'synthetic_singing',
+           'Angry',
+           'Sad',
+           'Happy',
+           'Surprise', 
+            'Fear', 
+            'Disgust', 
+            'Contempt', 
+            'Neutral'
+            ]
+
+
+args = transformers.Wav2Vec2Config() #finetuning_task='spef2feat_reg')
+args.dev = torch.device('cuda:0')
+args.dev2 = torch.device('cuda:0')
+
+# def _softmax(x):
+#     '''x : (batch, num_class)'''
+#     x -= x.max(1, keepdims=True)  # if all -400 then sum(exp(x)) = 0
+#     x = np.minimum(-100, x)
+#     x = np.exp(x)
+#     x /= x.sum(1, keepdims=True)
+#     return x
+
+def _softmax(x):
+    '''x : (batch, num_class)'''
+    x -= x.max(1, keepdims=True)  # if all -400 then sum(exp(x)) = 0
+    x = np.maximum(-100, x)
+    x = np.exp(x)
+    x /= x.sum(1, keepdims=True)
+    return x
+
+def _sigmoid(x):
+    '''x : (batch, num_class)'''
+    return 1 / (1 + np.exp(-x))    
+
+
+        # --
+    # ALL = anger, contempt, disgust, fear, happiness, neutral, no_agreement, other, sadness, surprise
+    # plot - unplesant emo 7x emo-categories [anger, contempt, disgust, fear, sadness] for artifical/sped-up/natural
+    # plot - pleasant emo [neutral, happiness, surprise]
+    # plot - Cubes Natural vs spedup   4x speed
+    # plot - synthesizer class audioset
+
+
+    # https://arxiv.org/pdf/2407.12229
+    #  https://arxiv.org/pdf/2312.05187
+    # https://arxiv.org/abs/2407.05407
+    # https://arxiv.org/pdf/2408.06577
+    # https://arxiv.org/pdf/2309.07405
+
+    
+# wavs are generated concat and plot time-series?
+
+# for mimic3/mimic3speed/human - concat all 77 and run timeseries with 7s hop 3s
+for long_audio in [
+    # 'mimic3.wav',
+    #                 'mimic3_speedup.wav',
+                    'human_770.wav',  # 'mimic3_all_77.wav', #
+                    'mimic3_770.wav',
+                    'mimic3_speed_770.wav'
+                    ]:
+    file_interface = f'timeseries_{long_audio.replace("/", "")}.pkl'
+    if not os.path.exists(file_interface):
+
+
+        print('_______________________________________\nProcessing\n', file_interface, '\n___________')
+
+
+
+        # CAT MSP
+
+        from transformers import AutoModelForAudioClassification
+        import types
+        def _infer(self, x):
+            '''x: (batch, audio-samples-16KHz)'''
+            x = (x + self.config.mean) / self.config.std  # plus
+            x = self.ssl_model(x, attention_mask=None).last_hidden_state
+            # pool
+            h = self.pool_model.sap_linear(x).tanh()
+            w = torch.matmul(h, self.pool_model.attention)
+            w = stylesoftmax(1)
+            mu = (x * w).sum(1)
+            x = torch.cat(
+                [
+                    mu,
+                    ((x * x * w).sum(1) - mu * mu).clamp(min=1e-7).sqrt()
+                ], 1)
+            return self.ser_model(x)
+
+        teacher_cat = AutoModelForAudioClassification.from_pretrained(
+            '3loi/SER-Odyssey-Baseline-WavLM-Categorical-Attributes',
+            trust_remote_code=True  # fun definitions see 3loi/SER-.. repo
+        ).to(args.dev2).eval()
+        teacher_cat.forward = types.MethodType(_infer, teacher_cat)
+        
+
+    
+        # Audioset & ADV
+
+        audioset_model = audonnx.load(audmodel.load('17c240ec-1.0.0'), device='cuda:0')
+        adv_model = audonnx.load(audmodel.load('90398682-2.0.0'), device='cuda:0')
+        
+        def process_function(x, sampling_rate, idx):
+            '''run audioset ct, adv
+
+               USE onnx teachers
+               
+               return [synth-speech, synth-singing, 7x, 3x adv] = 11
+            '''
+            
+            # x = x[None , :]  ASaHSuFDCN
+            #{0: 'Angry', 1: 'Sad', 2: 'Happy', 3: 'Surprise', 
+            #4: 'Fear', 5: 'Disgust', 6: 'Contempt', 7: 'Neutral'}
+            #tensor([[0.0015, 0.3651, 0.0593, 0.0315, 0.0600, 0.0125, 0.0319, 0.4382]])
+            logits_cat = teacher_cat(torch.from_numpy(x).to(args.dev)).cpu().detach().numpy()
+            # USE ALL CATEGORIES
+            # --
+            logits_audioset = audioset_model(x, 16000)['logits_sounds']
+            logits_audioset = logits_audioset[:, [7, 35]]  # speech synthesizer synthetic singing
+            # --
+            logits_adv = adv_model(x, 16000)['logits']
+            
+            cat = np.concatenate([logits_adv,
+                                  _sigmoid(logits_audioset),
+                                  _softmax(logits_cat)],
+                                  1)
+            print(cat)
+            return cat #logits_adv #model(signal, sampling_rate)['logits']
+        
+        interface = audinterface.Feature(
+            feature_names=LABELS,
+            process_func=process_function,
+            # process_func_args={'outputs': 'logits_scene'},
+            process_func_applies_sliding_window=False,
+            win_dur=40.0,
+            hop_dur=10.0,
+            sampling_rate=16000,
+            resample=True,
+            verbose=True,
+        )
+        df_pred = interface.process_file(long_audio)
+        df_pred.to_pickle(file_interface)
+    else:
+        print(file_interface, 'FOUND')
+        # df_pred = pd.read_pickle(file_interface)
+# ===============================================================================
+# V I S U A L S by loading all 3 pkl - mimic3 - speedup - human pd
+#
+# ===============================================================================
+
+
+preds  = {}
+SHORTEST_PD = 100000  # segments
+for long_audio in [
+    # 'mimic3.wav',
+    #                 'mimic3_speedup.wav',
+                    'human_770.wav',  # 'mimic3_all_77.wav', #
+                    'mimic3_770.wav',
+                    'mimic3_speed_770.wav'
+                    ]:
+    file_interface = f'timeseries_{long_audio.replace("/", "")}.pkl'
+    y = pd.read_pickle(file_interface)
+    preds[long_audio] = y
+    SHORTEST_PD = min(SHORTEST_PD, len(y))
+
+# clean indexes for plot
+
+for k,v in preds.items():
+    p = v[:SHORTEST_PD]  # TRuncate extra segments - human is slower than mimic3
+    # p = pd.read_pickle(student_file)
+    p.reset_index(inplace= True)
+    p.drop(columns=['file','start'], inplace=True)
+    p.set_index('end', inplace=True)
+    # p = p.filter(scene_classes) #['transport', 'indoor', 'outdoor'])
+    p.index = p.index.map(mapper = (lambda x: x.total_seconds()))
+    preds[k] = p
+
+    print(p, '\n\n\n\n \n')
+
+
+# Show plots by 2
+
+fig, ax = plt.subplots(nrows=10, ncols=2, figsize=(24, 24), gridspec_kw={'hspace': 0, 'wspace': .04})
+
+
+# ADV 
+
+time_stamp = preds['human_770.wav'].index.to_numpy()
+for j, dim in enumerate(['arousal', 
+                         'dominance', 
+                         'valence']):
+
+    # MIMIC3                      
+
+    ax[j, 0].plot(time_stamp, preds['mimic3_770.wav'][dim], 
+                color=(0,104/255,139/255), 
+                label='mean_1', 
+                linewidth=2)
+    ax[j, 0].fill_between(time_stamp,
+
+                    preds['mimic3_770.wav'][dim],
+                    preds['human_770.wav'][dim],
+
+                    color=(.2,.2,.2), 
+                    alpha=0.244)
+    if j == 0:                    
+        ax[j, 0].legend(['StyleTTS2 style mimic3',
+                        'StyleTTS2 style crema-d'], 
+                        prop={'size': 10}, 
+                        #  loc='lower right'
+                        )
+    ax[j, 0].set_ylabel(dim.lower(), color=(.4, .4, .4), fontsize=14)
+    
+    # TICK
+    ax[j, 0].set_ylim([1e-7, .9999])
+    # ax[j, 0].set_yticks([.25, .5,.75])
+    # ax[j, 0].set_yticklabels(['0.25', '.5', '0.75'])
+    ax[j, 0].set_xticklabels(['' for _ in ax[j, 0].get_xticklabels()])
+    ax[j, 0].set_xlim([time_stamp[0], time_stamp[-1]])
+
+
+   # MIMIC3   4x speed
+
+
+    ax[j, 1].plot(time_stamp, preds['mimic3_speed_770.wav'][dim], 
+                color=(0,104/255,139/255), 
+                label='mean_1', 
+                linewidth=2)
+    ax[j, 1].fill_between(time_stamp,
+
+                    preds['mimic3_speed_770.wav'][dim],
+                    preds['human_770.wav'][dim],
+
+                    color=(.2,.2,.2), 
+                    alpha=0.244)
+    if j == 0:                    
+        ax[j, 1].legend(['StyleTTS2 style mimic3   4x speed',
+                        'StyleTTS2 style crema-d'], 
+                        prop={'size': 10}, 
+                        #  loc='lower right'
+                        )
+
+
+    ax[j, 1].set_xlabel('767 Harvard Sentences (seconds)')
+
+
+
+    # TICK
+    ax[j, 1].set_ylim([1e-7, .9999])
+    # ax[j, 1].set_yticklabels(['' for _ in ax[j, 1].get_yticklabels()])
+    ax[j, 1].set_xticklabels(['' for _ in ax[j, 0].get_xticklabels()])
+    ax[j, 1].set_xlim([time_stamp[0], time_stamp[-1]])
+
+
+
+
+    ax[j, 0].grid()
+    ax[j, 1].grid()
+# CATEGORIE
+
+
+
+
+
+time_stamp = preds['human_770.wav'].index.to_numpy()
+for j, dim in enumerate(['Angry', 
+                         'Sad',
+                         'Happy',
+                         'Surprise', 
+                         'Fear',
+                         'Disgust', 
+                         'Contempt',
+                        #  'Neutral'
+                         ]):   # ASaHSuFDCN
+    j = j + 3  # skip A/D/V suplt                         
+
+    # MIMIC3                      
+
+    ax[j, 0].plot(time_stamp, preds['mimic3_770.wav'][dim], 
+                color=(0,104/255,139/255), 
+                label='mean_1', 
+                linewidth=2)
+    ax[j, 0].fill_between(time_stamp,
+
+                    preds['mimic3_770.wav'][dim],
+                    preds['human_770.wav'][dim],
+
+                    color=(.2,.2,.2), 
+                    alpha=0.244)
+    # ax[j, 0].legend(['StyleTTS2 style mimic3',
+    #                  'StyleTTS2 style crema-d'], 
+    #                  prop={'size': 10}, 
+    #                 #  loc='upper left'
+    # )
+
+
+    ax[j, 0].set_ylabel(dim.lower(), color=(.4, .4, .4), fontsize=14)
+
+    # TICKS
+    ax[j, 0].set_ylim([1e-7, .9999])
+    ax[j, 0].set_xlim([time_stamp[0], time_stamp[-1]])
+    ax[j, 0].set_xticklabels(['' for _ in ax[j, 0].get_xticklabels()])
+    ax[j, 0].set_xlabel('767 Harvard Sentences (seconds)', fontsize=16, color=(.4,.4,.4))
+
+
+   # MIMIC3   4x speed
+
+
+    ax[j, 1].plot(time_stamp, preds['mimic3_speed_770.wav'][dim], 
+                color=(0,104/255,139/255), 
+                label='mean_1', 
+                linewidth=2)
+    ax[j, 1].fill_between(time_stamp,
+
+                    preds['mimic3_speed_770.wav'][dim],
+                    preds['human_770.wav'][dim],
+
+                    color=(.2,.2,.2), 
+                    alpha=0.244)
+    # ax[j, 1].legend(['StyleTTS2 style mimic3   4x speed',
+    #                  'StyleTTS2 style crema-d'], 
+    #                  prop={'size': 10},
+    #                 #  loc='upper left'
+    # )
+    ax[j, 1].set_xlabel('767 Harvard Sentences (seconds)', fontsize=16, color=(.4,.4,.4))
+    ax[j, 1].set_ylim([1e-7, .999])
+    # ax[j, 1].set_yticklabels(['' for _ in ax[j, 1].get_yticklabels()])
+    ax[j, 1].set_xticklabels(['' for _ in ax[j, 1].get_xticklabels()])
+    ax[j, 1].set_xlim([time_stamp[0], time_stamp[-1]])
+    
+
+
+
+
+
+    ax[j, 0].grid()
+    ax[j, 1].grid()
+
+
+
+plt.savefig(f'valence_tts.pdf', bbox_inches='tight')
+plt.close()
+