train.py

"""XTTS fine-tuning on HF T4 GPU with auto-upload."""
import os, sys, subprocess

# Monkey-patch TTS to skip CPML license prompt (global solution)
import TTS.utils.manage as _tts_manage
_tts_manage.ModelManager.ask_tos = lambda self, path: True

import torch, gc, glob, csv, time

# Set memory optimization for T4 16GB
os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True,max_split_size_mb:32"

from TTS.api import TTS

print(f"CUDA: {torch.cuda.is_available()}", flush=True)
if torch.cuda.is_available():
    print(f"GPU: {torch.cuda.get_device_name(0)}", flush=True)
    print(f"VRAM: {torch.cuda.get_device_properties(0).total_memory / 1e9:.1f} GB", flush=True)

gc.collect()
torch.cuda.empty_cache()

tts = TTS("tts_models/multilingual/multi-dataset/xtts_v2")
print("Model loaded!", flush=True)

# Read dataset
def read_csv(path):
    rows = []
    with open(path, newline='', encoding='utf-8') as f:
        reader = csv.DictReader(f, delimiter='|')
        for row in reader:
            rows.append(row)
    return rows

train_rows = read_csv('train.csv')
eval_rows = read_csv('eval.csv')
print(f"Train: {len(train_rows)}, Eval: {len(eval_rows)}", flush=True)

# Copy pre-downloaded XTTS model files from TTS cache to training dir
TTS_CACHE = os.path.expanduser("~/.local/share/tts/tts_models--multilingual--multi-dataset--xtts_v2")
XTTS_DIR = os.path.join("ft_output", "run", "training", "XTTS_v2.0_original_model_files")
os.makedirs(XTTS_DIR, exist_ok=True)

if os.path.exists(TTS_CACHE):
    import shutil
    for f in os.listdir(TTS_CACHE):
        src = os.path.join(TTS_CACHE, f)
        dst = os.path.join(XTTS_DIR, f)
        if not os.path.exists(dst) and os.path.isfile(src):
            print(f"  Copying {f} from cache ({os.path.getsize(src)/1e6:.0f} MB)", flush=True)
            shutil.copy2(src, dst)

# Check what's missing and download with retries
import urllib.request
MISSING = [f for f in ["dvae.pth", "mel_stats.pth", "model.pth", "config.json", "vocab.json"]
           if not os.path.exists(os.path.join(XTTS_DIR, f)) or os.path.getsize(os.path.join(XTTS_DIR, f)) < 1000]

if MISSING:
    BASE_URL = "https://coqui.gateway.scarf.sh/hf-coqui/XTTS-v2/main"
    for fname in MISSING:
        url = f"{BASE_URL}/{fname}"
        dest = os.path.join(XTTS_DIR, fname)
        for attempt in range(5):
            try:
                print(f"  Downloading {fname}... (attempt {attempt+1}/5)", flush=True)
                urllib.request.urlretrieve(url, dest)
                print(f"  {fname} OK ({os.path.getsize(dest)/1e6:.0f} MB)", flush=True)
                break
            except Exception as e:
                print(f"  Retry {attempt+1}: {str(e)[:80]}", flush=True)
                if attempt == 4:
                    print(f"  Gave up on {fname}, will let train_gpt handle", flush=True)
                import time; time.sleep(3)

# Aggressive memory cleanup before training
gc.collect()
torch.cuda.empty_cache()

# Train with reduced memory footprint for T4 16GB
output_path = "ft_output"
os.makedirs(output_path, exist_ok=True)

from TTS.demos.xtts_ft_demo.utils.gpt_train import train_gpt

start = time.time()
train_gpt(
    language="es",
    num_epochs=10,
    batch_size=1,
    grad_acumm=4,
    train_csv="train.csv",
    eval_csv="eval.csv",
    output_path=output_path,
    max_audio_length=255995,
)
elapsed = time.time() - start

# Find best model
best = None
for f in sorted(glob.glob(os.path.join(output_path, "**", "best_model*.pth"), recursive=True)):
    best = f

print(f"\nTraining completed in {elapsed:.0f}s ({elapsed/60:.1f} min)", flush=True)
if best:
    print(f"Best model: {best}", flush=True)
    print(f"Size: {os.path.getsize(best)/1e6:.0f} MB", flush=True)

    # Upload best model to HF Space using huggingface_hub
    print("Uploading model to HF Space...", flush=True)
    
    try:
        from huggingface_hub import HfApi, CommitOperationAdd
        hf_token = os.environ.get("HF_TOKEN", "")
        if not hf_token:
            # Try reading from file
            try:
                with open("/app/.hf_token") as f:
                    hf_token = f.read().strip()
            except:
                pass
        
        if hf_token:
            api = HfApi()
            api.upload_file(
                path_or_fileobj=best,
                path_in_repo="model/best_model.pth",
                repo_id="Converso72/julio-xtts-voice",
                repo_type="model",
                token=hf_token
            )
            print(f"  Model uploaded: model/best_model.pth", flush=True)
        else:
            print("  No HF_TOKEN available, saving model locally", flush=True)
    except Exception as e:
        print(f"  Upload failed: {e}", flush=True)

    print("Model upload complete!", flush=True)

print("DONE!", flush=True)