Upload 3 files

Training and speech generation scripts.

expandnumbers.py

@@ -0,0 +1,63 @@
+import random
+
+
+def number_in_words(getal):
+    eenheden = ["", "een", "twee", "drie", "vier", "vijf", "zes", "zeven", "acht", "negen"]
+    tientallen = ["", "", "twintig", "dertig", "veertig", "vijftig", "zestig", "zeventig", "tachtig", "negentig"]
+
+    miljoentallen = ""
+    duizendtallen = ""
+    if getal >= 1000000:
+        miljoentallen = number_to_hundreds(getal // 1000000) + "miljoen "
+        getal = getal % 1000000
+
+    if getal >= 1000:
+        duizendtallen = number_to_hundreds(getal // 1000) + "duizend "
+        getal = getal % 1000
+
+    return miljoentallen + duizendtallen + number_to_hundreds(getal)
+
+
+def number_to_hundreds(num):
+    eenheden = ["", "een", "twee", "drie", "vier", "vijf", "zes", "zeven", "acht", "negen", "tien"]
+    tientallen = ["", "", "twintig", "dertig", "veertig", "vijftig", "zestig", "zeventig", "tachtig", "negentig"]
+    res = ""
+
+    if num >= 100:
+        res = eenheden[num // 100] + "honderd"
+        num = num % 100
+
+    if 10 < num < 20:
+        special_cases = {
+            11: "elf", 12: "twaalf", 13: "dertien", 14: "veertien", 15: "vijftien", 
+            16: "zestien", 17: "zeventien", 18: "achttien", 19: "negentien"
+        }
+        res += special_cases[num]
+    elif num >= 20:
+        if num % 10 != 0:
+            res += eenheden[num % 10] + "en"
+        res += tientallen[num // 10]
+    # elif num == 10:
+    #     res += eenheden[0]
+    else:
+        print('num', num)
+        res += eenheden[num]
+
+    return res
+
+
+
+if __name__ == "__main__":
+ 
+    for d in range(10):
+
+        case = {}
+
+        getal = random.randint(1, 1000)
+
+        print('getal', getal)
+
+        woorden = number_in_words(getal)
+        print(woorden)
+
+

generateSpeechT5ttsHF.ipynb

@@ -0,0 +1,151 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "58146025",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "model & processor loaded\n",
+      "speaker embeddings & vocoder loaded\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "os.environ[\"WANDB_MODE\"] = \"disabled\"\n",
+    "\n",
+    "import torch\n",
+    "\n",
+    "import soundfile as sf\n",
+    "import sounddevice as sd\n",
+    "\n",
+    "from datasets import load_dataset \n",
+    "\n",
+    "from transformers import SpeechT5ForTextToSpeech, SpeechT5Processor\n",
+    "from transformers import SpeechT5HifiGan\n",
+    "\n",
+    "\n",
+    "MODEL_NAME = \"microsoft/speecht5_tts\"\n",
+    "\n",
+    "CACHE_DIR = \"D:/LanguageModels/cache\"\n",
+    "DATASET_DIR = \"D:/LanguageModels/dataset/\"\n",
+    "AUDIO_DIR = \"D:/LanguageModels/dataset/audio/\"\n",
+    "\n",
+    "finetuned = True\n",
+    "\n",
+    "if finetuned:\n",
+    "    model = SpeechT5ForTextToSpeech.from_pretrained(\"D:/LanguageModels/ftT5modelGetallen\")\n",
+    "    processor = SpeechT5Processor.from_pretrained(\"D:/LanguageModels/ftT5processorGetallen\")\n",
+    "else:\n",
+    "    model = SpeechT5ForTextToSpeech.from_pretrained(MODEL_NAME , cache_dir=CACHE_DIR)\n",
+    "    processor = SpeechT5Processor.from_pretrained(MODEL_NAME , cache_dir=CACHE_DIR)\n",
+    "\n",
+    "print('model & processor loaded')\n",
+    "\n",
+    "embeddings_dataset = load_dataset(\"Matthijs/cmu-arctic-xvectors\", split=\"validation\" , cache_dir=CACHE_DIR)\n",
+    "speaker_embeddings = torch.tensor(embeddings_dataset[7306][\"xvector\"]).unsqueeze(0)\n",
+    "vocoder = SpeechT5HifiGan.from_pretrained(\"microsoft/speecht5_hifigan\" , cache_dir=CACHE_DIR)\n",
+    "\n",
+    "print('speaker embeddings & vocoder loaded')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "e0c771b3",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "num 9\n",
+      "negen\n",
+      "inputs[\"input_ids\"] tensor([[ 4,  9,  5, 21,  5,  9,  2]])\n",
+      "spectrogram tensor([[-6.1973, -6.0982, -6.2766,  ..., -7.5271, -7.3350, -7.0105],\n",
+      "        [-5.0723, -5.2716, -5.1717,  ..., -5.6201, -5.4415, -5.4130],\n",
+      "        [-4.9598, -5.0752, -4.9666,  ..., -5.1151, -4.9876, -5.0876],\n",
+      "        ...,\n",
+      "        [-4.9689, -4.7987, -4.7155,  ..., -4.8989, -4.9458, -4.9992],\n",
+      "        [-4.7762, -4.8095, -4.8122,  ..., -4.8018, -4.8213, -4.8536],\n",
+      "        [-4.4982, -4.5212, -4.4879,  ..., -4.7741, -4.8168, -4.8919]])\n",
+      "spectrogram type <class 'torch.Tensor'>\n",
+      "speech tensor([ 7.7491e-06,  1.3468e-05, -1.8129e-06,  ..., -9.3731e-06,\n",
+      "         1.2687e-05,  3.4955e-07])\n",
+      "speech shape torch.Size([14848])\n",
+      "Speech synthesis complete and saved for number negen to ./output/negen_T5modelFineGetallen.wav\n"
+     ]
+    }
+   ],
+   "source": [
+    "from expandnumbers import getal_in_woorden\n",
+    "import re\n",
+    "\n",
+    "\n",
+    "# Replace numbers with words\n",
+    "def replace_numbers(text):\n",
+    "    return re.sub(r'\\d+', lambda x: getal_in_woorden(int(x.group())), text)\n",
+    "\n",
+    "\n",
+    "number = '9'\n",
+    "\n",
+    "number = replace_numbers(number)\n",
+    "print(number)\n",
+    "\n",
+    "inputs = processor(text = number, return_tensors=\"pt\")\n",
+    "print('inputs[\"input_ids\"]' , inputs[\"input_ids\"] )\n",
+    "\n",
+    "spectrogram = model.generate_speech(inputs[\"input_ids\"], speaker_embeddings)\n",
+    "\n",
+    "print('spectrogram' , spectrogram)\n",
+    "print('spectrogram type' , type(spectrogram))\n",
+    "\n",
+    "speech = model.generate_speech(inputs[\"input_ids\"], speaker_embeddings, vocoder=vocoder)\n",
+    "\n",
+    "print('speech' , speech)\n",
+    "print('speech shape' , speech.shape)\n",
+    "\n",
+    "# Play the sound\n",
+    "sd.play(speech, 16000)\n",
+    "sd.wait()  # Wait until playback finishes\n",
+    "\n",
+    "outfilename = ''\n",
+    "\n",
+    "if finetuned:\n",
+    "    outfilename = './output/' + number + '_T5modelFineGetallen.wav'\n",
+    "else:\n",
+    "    outfilename = './output/' + number + '_T5modelOrigGetallen.wav'\n",
+    "\n",
+    "sf.write(outfilename, speech, 16000)\n",
+    "\n",
+    "print(\"Speech synthesis complete and saved for number \" + number + ' to ' + outfilename)\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

trainSpeechT5ttsGetallen.py

@@ -0,0 +1,354 @@
+
+import os
+os.environ["WANDB_MODE"] = "disabled"
+
+import tensorflow as tf
+import torch
+import torchaudio
+
+from transformers import TrainingArguments, Trainer
+from transformers import SpeechT5HifiGan
+from transformers import TrainerCallback
+
+from datasets import load_dataset , Audio
+from datasets import concatenate_datasets
+
+# from peft import get_peft_model, LoraConfig
+
+MODEL_NAME = "microsoft/speecht5_tts"
+
+CACHE_DIR = "D:/LanguageModels/cache"
+
+DATASET_DIR = "./numberaudiodata/"
+AUDIO_DIR = "./numberaudiodata/"
+
+dataset = load_dataset("json", data_files = DATASET_DIR + "trainNL.json")
+
+print('dataset' , dataset)
+
+dataset = dataset["train"]  # Ensure correct split selection
+
+print('dataset["train"]' , dataset)
+
+# Replicate the dataset to make it 10 times larger
+datasets = []
+
+for d in range(10):
+    datasets.append(dataset)
+
+dataset = concatenate_datasets(datasets)
+
+print(len(dataset))  # Check new dataset size
+
+exit()
+
+
+# gender_mapping = {"number1.wav": "male", "number2.wav": "female"}  # Example mapping
+# dataset = dataset.map(lambda example: {**example, "gender": gender_mapping.get(example["audio"], "unknown")})
+print(type(dataset))
+print(len(dataset))
+print(dataset[0])
+
+audio_id = 1
+
+def update_example(example, audio_id):
+    example['audio_id'] = audio_id
+    example['language'] = 9
+    example['gender'] = 'female'
+    example['speaker_id'] = '1122'
+    example['is_gold_transcript'] = True
+    example['accent'] = 'None'
+
+    waveform, sampling_rate = torchaudio.load(AUDIO_DIR + example['audiofile'])
+    # Convert to NumPy
+    audio_array = waveform.numpy()
+    # print(audio_array.flatten())
+    example['audio'] = {}
+    example['audio']['array'] = audio_array.flatten()
+    example['audio']['sampling_rate'] = sampling_rate
+
+    return example
+
+# Use `enumerate` with `map()` to update dataset
+dataset = dataset.map(lambda example, idx: update_example(example, idx + 1), with_indices=True)
+
+dataset = dataset.cast_column("audio", Audio(sampling_rate=16000))
+
+print(dataset[3])
+# exit(0)
+
+from transformers import SpeechT5ForTextToSpeech, SpeechT5Processor
+
+processor = SpeechT5Processor.from_pretrained(MODEL_NAME , cache_dir=CACHE_DIR)
+
+
+
+def prepare_dataset(example):
+
+    audio = example["audio"]
+
+    example = processor(
+        text=example["text"],
+        audio_target=audio["array"],
+        sampling_rate=audio["sampling_rate"],
+        return_attention_mask=False,
+    )
+
+    # strip off the batch dimension
+    example["labels"] = example["labels"][0]
+
+    embeddings_dataset = load_dataset("Matthijs/cmu-arctic-xvectors", split="validation" , cache_dir=CACHE_DIR)
+
+    speaker_embeddings = torch.tensor(embeddings_dataset[7306]["xvector"])
+
+    example["speaker_embeddings"] = speaker_embeddings
+
+    return example
+
+
+processed_example = prepare_dataset(dataset[0])
+print(list(processed_example.keys()))
+print(processed_example["speaker_embeddings"].shape)
+print('type(processed_example["labels"])' , type(processed_example["labels"]))
+
+dataset = dataset.map(prepare_dataset, remove_columns=dataset.column_names)
+
+print('dataset.column_names' , dataset.column_names)
+print('training data negen input_ids' , dataset[8]['input_ids'])
+print('training data negen labels' , dataset[8]['labels'][0])
+
+
+loadedspectrogram = dataset[8]['labels']
+print('loadedspectrogram is ' , type(loadedspectrogram))
+
+# Ensure spectrogram is a PyTorch tensor
+spectrogram = torch.tensor(loadedspectrogram)
+
+# spectrogram = torch.tensor(loadedspectrogram).unsqueeze(0)  # Add batch dimension if needed
+
+vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan" , cache_dir=CACHE_DIR)
+# speech = model.generate_speech(inputs["input_ids"], speaker_embeddings, vocoder=vocoder)
+
+print('vocoder loaded')
+
+# Generate waveform from spectrogram
+speech = vocoder(spectrogram).detach()
+
+print('speech' , speech)
+print('speech shape' , speech.shape)
+
+
+
+# Play the sound
+# sd.play(speech, 16000)
+# sd.wait()  # Wait until playback finishes
+
+# sf.write('./output/loaded_negenGetallen.wav', speech, 16000)
+
+
+# dataset = dataset.train_test_split(test_size=0.1)
+
+from dataclasses import dataclass
+from typing import Any, Dict, List, Union
+
+
+@dataclass
+class TTSDataCollatorWithPadding:
+    processor: Any
+
+    def __call__(
+        self, features: List[Dict[str, Union[List[int], torch.Tensor]]]
+    ) -> Dict[str, torch.Tensor]:
+        
+        input_ids = [{"input_ids": feature["input_ids"]} for feature in features]
+        label_features = [{"input_values": feature["labels"]} for feature in features]
+        
+        speaker_features = [feature["speaker_embeddings"] for feature in features]
+
+        # collate the inputs and targets into a batch
+        batch = processor.pad(
+            input_ids=input_ids, labels=label_features, return_tensors="pt"
+        )
+
+        # replace padding with -100 to ignore loss correctly
+        batch["labels"] = batch["labels"].masked_fill(
+            batch.decoder_attention_mask.unsqueeze(-1).ne(1), -100
+        )
+
+        # not used during fine-tuning
+        del batch["decoder_attention_mask"]
+
+        # round down target lengths to multiple of reduction factor
+        if model.config.reduction_factor > 1:
+            target_lengths = torch.tensor(
+                [len(feature["input_values"]) for feature in label_features]
+            )
+            target_lengths = target_lengths.new(
+                [
+                    length - length % model.config.reduction_factor
+                    for length in target_lengths
+                ]
+            )
+            max_length = max(target_lengths)
+            batch["labels"] = batch["labels"][:, :max_length]
+
+        # also add in the speaker embeddings
+        batch["speaker_embeddings"] = torch.tensor(speaker_features)
+
+        # print('batch["input_ids"]' , batch["input_ids"][3])
+        # print('batch["labels"]' , batch["labels"][3])
+
+        return batch
+    
+
+data_collator = TTSDataCollatorWithPadding(processor=processor)
+
+
+training = True
+
+
+class SaveBestModelCallback(TrainerCallback):
+    def __init__(self, model, processor, save_path, best_loss=float("inf")):
+        self.model = model
+        self.processor = processor
+        self.save_path = save_path
+        self.best_loss = best_loss
+
+    def on_step_end(self, args, state, control, **kwargs):
+        if state.log_history and "loss" in state.log_history[-1]: 
+            current_loss = state.log_history[-1]["loss"]
+            if current_loss < 1.00 and current_loss < self.best_loss:
+                self.best_loss = current_loss
+                print(f"New best loss: {self.best_loss} - Saving model...")
+                self.model.save_pretrained(self.save_path)
+                self.processor.save_pretrained(self.save_path)
+
+
+import torch.nn as nn
+
+
+class MSETrainer(Trainer):
+
+    def compute_loss(self, model, inputs, return_outputs=False, num_items_in_batch=None):
+
+        labels = inputs.pop("labels", None)  # Extract target speech outputs
+        if labels is None:
+            raise ValueError("Labels are missing in inputs!")
+        else:
+            print('labels' , labels)
+
+# Ensure inputs are valid before passing to model
+        for key, value in inputs.items():
+            if value is None:
+                raise ValueError(f"Input '{key}' is None! Check preprocessing.")
+
+        print(' inputs are valid before passing to model') 
+
+        outputs = model(**inputs)  # Forward pass to get predictions
+
+        print('outputs' , outputs) 
+
+        loss_fn = nn.MSELoss()  # Define MSE loss function
+        loss = loss_fn(outputs.logits, labels)  # Compute loss between predictions & labels
+
+        print('loss' , loss) 
+
+        return (loss, outputs) if return_outputs else loss
+
+    # def compute_loss(self, model, inputs, return_outputs=False, num_items_in_batch=None):
+
+    #     # labels = inputs.pop("labels")  # Extract target speech outputs
+    #     # outputs = model(**inputs)  # Model prediction
+    #     loss = super().compute_loss(model, inputs, return_outputs)
+
+    #     # loss = torch.nn.MSELoss()(outputs.logits, labels)  # Compute MSE loss
+
+    #     return  loss
+
+
+
+if training:
+
+    print('now training')
+
+
+    model = SpeechT5ForTextToSpeech.from_pretrained(MODEL_NAME , cache_dir=CACHE_DIR)
+
+    print(model.config)
+    targetmodules = []
+    nall = 0
+    ntarget = 0
+    for name, module in model.named_modules():
+        nall += 1
+        if 'speech_decoder_postnet' in name:
+            ntarget +=1
+            targetmodules.append(name)
+            print(name, "->", type(module))
+
+    print('nall' , nall , 'ntarget' , ntarget)
+
+  
+    print(dir(model))
+
+    print(model.named_parameters())
+
+    # exit()
+
+    # Fine-tune only the postnet
+    for param in model.parameters():
+        param.requires_grad = False  # Freeze all layers
+    for param in model.speech_decoder_postnet.parameters():
+        param.requires_grad = True  # Unfreeze postnet
+
+    # config = LoraConfig(
+    #     r=8,  # Rank of LoRA matrices
+    #     lora_alpha=32,
+    #     lora_dropout=0.1,
+    #     # target_modules=targetmodules  # Apply LoRA to decoder layers
+    #     target_modules = ["speech_decoder_postnet.feat_out", "speech_decoder_postnet.prob_out"]  # Example names
+
+    #     # target_modules=["decoder.block", "decoder.attention", "speech_decoder_prenet"],  # Adjust based on model structure
+    #     # target_modules=["decoder.layers"],  # Apply LoRA to decoder layers
+    # )
+
+    # peftmodel = get_peft_model(model, config)
+
+    training_args = TrainingArguments(
+        output_dir="D:/LanguageModels/out_T5tts",
+        run_name="tts_exp", 
+        per_device_train_batch_size=32,
+        gradient_accumulation_steps=1,
+        num_train_epochs=100,
+        save_steps=100,
+        save_total_limit=2,
+        # logging_dir="D:/LanguageModels/logs",
+        logging_steps=1,
+        learning_rate=0.001,
+        optim="adamw_torch_fused",
+        eval_strategy="no",
+        # eval_strategy="steps",
+        eval_steps=500,
+        weight_decay=0.0,
+        fp16=True
+    )
+
+    # from transformers import DataCollatorWithPadding
+
+    # collator = DataCollatorWithPadding(tokenizer=processor.tokenizer, padding=True)
+
+    trainer = Trainer(
+        model = model,
+        args = training_args,
+        train_dataset = dataset,
+        data_collator = data_collator,
+        callbacks=[SaveBestModelCallback(model, processor, "D:/LanguageModels/ftT5modelDutchNumbers")]
+    )
+
+    trainer.train()
+
+
+    # model.save_pretrained("D:/LanguageModels/ftT5modelGetallen")
+    # processor.save_pretrained("D:/LanguageModels/ftT5processorGetallen")
+
+
+