Upload 3 files

loraV2/finetune_elise_fadeout.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+# Single-speaker fine-tuning script for VibeVoice-1.5B on Elise dataset
+# WITH FADEOUT AND PADDING - ensures all audio has clean endings
+# No voice prompts - pure text-to-speech for Elise voice only
+
+echo "Single-speaker fine-tuning with fadeout dataset..."
+echo "All audio files have 100ms fadeout + 250ms padding"
+echo "NO voice prompts - training pure Elise TTS model"
+
+python -m src.finetune_vibevoice_lora \
+    --model_name_or_path . \
+    --train_jsonl jinsaryko_elise_fadeout/elise_train_split.jsonl \
+    --validation_jsonl jinsaryko_elise_fadeout/elise_val.jsonl \
+    --text_column_name text \
+    --audio_column_name audio \
+    --output_dir finetune_elise_fadeout \
+    --per_device_train_batch_size 4 \
+    --gradient_accumulation_steps 8 \
+    --learning_rate 2.5e-5 \
+    --num_train_epochs 4 \
+    --logging_steps 10 \
+    --save_steps 100 \
+    --eval_steps 100 \
+    --report_to none \
+    --remove_unused_columns False \
+    --bf16 True \
+    --do_train \
+    --do_eval \
+    --gradient_clipping \
+    --gradient_checkpointing False \
+    --ddpm_batch_mul 2 \
+    --diffusion_loss_weight 1.4 \
+    --train_diffusion_head True \
+    --ce_loss_weight 0.04 \
+    --voice_prompt_drop_rate 1.0 \
+    --lora_target_modules q_proj,k_proj,v_proj,o_proj,gate_proj,up_proj,down_proj \
+    --lr_scheduler_type cosine \
+    --warmup_ratio 0.03 \
+    --max_grad_norm 0.8 \
+    --max_length 4096
+
+echo "Single-speaker fine-tuning with fadeout complete!"
+echo "Model trained on audio with smooth fadeouts - no abrupt endings!"

loraV2/prepare_jinsaryko_elise_fadeout.py

@@ -0,0 +1,112 @@
+import json
+import os
+from datasets import load_dataset
+import soundfile as sf
+import numpy as np
+from tqdm import tqdm
+
+# Create output directories
+os.makedirs("jinsaryko_elise_fadeout", exist_ok=True)
+os.makedirs("jinsaryko_elise_fadeout/wavs", exist_ok=True)
+
+def apply_fadeout_and_padding(audio_array, sr, fadeout_duration=0.1, silence_duration=0.25):
+    """
+    Apply fadeout and add silence padding to audio
+
+    Args:
+        audio_array: numpy array of audio samples
+        sr: sample rate
+        fadeout_duration: duration of fadeout in seconds (default 0.1s = 100ms)
+        silence_duration: duration of silence to add after fadeout (default 0.25s = 250ms)
+
+    Returns:
+        processed audio array
+    """
+    # Calculate sample counts
+    fadeout_samples = int(fadeout_duration * sr)
+    silence_samples = int(silence_duration * sr)
+
+    # Make sure we don't fade out more than the audio length
+    if fadeout_samples > len(audio_array):
+        fadeout_samples = len(audio_array) // 2
+
+    # Create fadeout curve (linear fade)
+    fade_curve = np.linspace(1.0, 0.0, fadeout_samples)
+
+    # Apply fadeout to the last part of the audio
+    audio_with_fade = audio_array.copy()
+    audio_with_fade[-fadeout_samples:] *= fade_curve
+
+    # Add silence padding
+    silence = np.zeros(silence_samples, dtype=audio_array.dtype)
+    audio_padded = np.concatenate([audio_with_fade, silence])
+
+    return audio_padded
+
+# Load the Jinsaryko/Elise dataset
+print("Loading Jinsaryko/Elise dataset...")
+dataset = load_dataset("Jinsaryko/Elise")
+
+# Since it's a single speaker dataset, we'll use voice_prompt_drop_rate=1.0 as recommended
+# But we still need to format the text with Speaker 0: prefix
+
+jsonl_data = []
+
+print("Processing audio files with fadeout and padding...")
+for idx, sample in enumerate(tqdm(dataset['train'])):
+    # Format text with Speaker 0: prefix
+    original_text = sample['text']
+    formatted_text = f"Speaker 0: {original_text}"
+
+    # Get audio data
+    audio = sample['audio']
+    audio_array = audio['array'].astype(np.float32)
+    sampling_rate = audio['sampling_rate']
+
+    # Apply fadeout and padding
+    processed_audio = apply_fadeout_and_padding(
+        audio_array,
+        sampling_rate,
+        fadeout_duration=0.1,  # 100ms fadeout
+        silence_duration=0.25  # 250ms silence
+    )
+
+    # Save processed audio file
+    audio_filename = f"jinsaryko_elise_fadeout/wavs/sample_{idx:06d}.wav"
+    sf.write(audio_filename, processed_audio, sampling_rate)
+
+    # Add to JSONL
+    jsonl_entry = {
+        "text": formatted_text,
+        "audio": os.path.abspath(audio_filename)
+    }
+    jsonl_data.append(jsonl_entry)
+
+# Save JSONL file
+print("Saving JSONL files...")
+with open("jinsaryko_elise_fadeout/elise_train.jsonl", "w") as f:
+    for entry in jsonl_data:
+        f.write(json.dumps(entry) + "\n")
+
+# Create a small validation set (5% of data)
+val_size = int(0.05 * len(jsonl_data))
+train_data = jsonl_data[:-val_size]
+val_data = jsonl_data[-val_size:]
+
+with open("jinsaryko_elise_fadeout/elise_train_split.jsonl", "w") as f:
+    for entry in train_data:
+        f.write(json.dumps(entry) + "\n")
+
+with open("jinsaryko_elise_fadeout/elise_val.jsonl", "w") as f:
+    for entry in val_data:
+        f.write(json.dumps(entry) + "\n")
+
+print(f"\nDataset with fadeout prepared!")
+print(f"Total samples: {len(jsonl_data)}")
+print(f"Training samples: {len(train_data)}")
+print(f"Validation samples: {len(val_data)}")
+print(f"Files saved in jinsaryko_elise_fadeout/")
+print(f"\nAudio processing applied:")
+print(f"- 100ms fadeout at the end of each clip")
+print(f"- 250ms silence padding after fadeout")
+print(f"- All audio files now have smooth endings!")

loraV2/test_fadeout_model.py

@@ -0,0 +1,111 @@
+import sys
+import os
+sys.path.append(os.path.join(os.path.dirname(__file__), 'src'))
+
+import torch
+from vibevoice.modular.modeling_vibevoice_inference import VibeVoiceForConditionalGenerationInference
+from vibevoice.processor.vibevoice_processor import VibeVoiceProcessor
+from peft import PeftModel
+import json
+
+print("Loading fadeout model (4 epochs, fadeout + padding dataset)...")
+model = VibeVoiceForConditionalGenerationInference.from_pretrained(
+    ".",
+    torch_dtype=torch.bfloat16,
+    device_map="cuda",
+    attn_implementation="flash_attention_2"
+)
+
+# Load the fadeout model weights
+model.model.language_model = PeftModel.from_pretrained(
+    model.model.language_model,
+    "finetune_elise_fadeout/lora"
+)
+
+diffusion_state = torch.load("finetune_elise_fadeout/lora/diffusion_head_full.bin", map_location="cpu")
+model.model.prediction_head.load_state_dict(diffusion_state)
+
+processor = VibeVoiceProcessor.from_pretrained("src/vibevoice/processor")
+model.eval()
+
+# Use optimal settings
+model.set_ddpm_inference_steps(num_steps=20)
+
+# Get a dummy voice sample (model ignores this due to voice_prompt_drop_rate=1.0)
+with open("jinsaryko_elise_fadeout/elise_train_split.jsonl", 'r') as f:
+    voice_data = json.loads(f.readline())
+    dummy_voice_path = voice_data['audio']
+
+print(f"\nUsing dummy voice (ignored): {os.path.basename(dummy_voice_path)}")
+print("Testing fadeout model with various length statements...\n")
+
+# Test sentences - short, medium, and long (ending with ellipses to prevent cutoffs)
+test_sentences = [
+    # Short statements
+    "Hello!",
+    "Good morning everyone...",
+    "Welcome to my channel...",
+    "Thanks for watching!",
+
+    # Medium statements
+    "Today we're going to learn something amazing together...",
+    "I'm really excited to share this with all of you...",
+    "Let me show you how this incredible feature works...",
+    "Have you ever wondered about the mysteries of the universe?",
+
+    # Long statements
+    "Welcome back to the channel! Today I have something really special to share with you, and I think you're going to absolutely love what we're about to explore together...",
+    "Throughout my journey of learning and discovery, I've come across many fascinating concepts, but this one in particular has completely transformed the way I think about technology and innovation...",
+    "The beauty of machine learning lies not just in its complexity, but in how it can bring seemingly impossible ideas to life, creating experiences that were once confined to the realm of science fiction...",
+    "As we dive deeper into this topic, I want you to imagine the endless possibilities that await us, and consider how these advancements might shape our future in ways we can barely comprehend today..."
+]
+
+os.makedirs("test_fadeout_output", exist_ok=True)
+
+for i, text in enumerate(test_sentences):
+    length = "short" if len(text) < 30 else "medium" if len(text) < 100 else "long"
+    print(f"\n[{i+1}/{len(test_sentences)}] [{length.upper()}] {text}")
+
+    prompt = f"Speaker 0: {text}"
+
+    inputs = processor(
+        text=[prompt],
+        voice_samples=[[dummy_voice_path]],  # Dummy voice (ignored by model)
+        return_tensors="pt"
+    )
+
+    for k, v in inputs.items():
+        if torch.is_tensor(v):
+            inputs[k] = v.to("cuda")
+
+    outputs = model.generate(
+        **inputs,
+        cfg_scale=2.0,
+        tokenizer=processor.tokenizer,
+        generation_config={'do_sample': False},
+        verbose=False
+    )
+
+    if outputs.speech_outputs and outputs.speech_outputs[0] is not None:
+        audio = outputs.speech_outputs[0]
+
+        # Save with light padding (model already has fadeout from training)
+        silence = torch.zeros_like(audio[..., :4800])  # 200ms padding
+        padded = torch.cat([audio, silence], dim=-1)
+
+        output_path = f"test_fadeout_output/test_{i:02d}_{length}.wav"
+        processor.save_audio(padded, output_path)
+
+        duration = audio.shape[-1] / 24000
+        print(f"  ✓ Generated {duration:.2f}s → {output_path}")
+
+print("\n" + "="*60)
+print("Fadeout model test complete!")
+print("Files saved in test_fadeout_output/")
+print("\nModel stats:")
+print("- 4 epochs on fadeout dataset (100ms fade + 250ms padding)")
+print("- Final CE loss: ~5.25")
+print("- Final Diffusion loss: ~0.559")
+print("- voice_prompt_drop_rate: 1.0 (no voice prompts)")
+print("- All training audio had smooth fadeouts!")
+print("="*60)