src/diffusion/train/train.py

import os
os.environ["TOKENIZERS_PARALLELISM"] = "false"

import shutil, sys, logging, yaml, hydra
from pathlib import Path
from os.path import join as pjoin
from omegaconf import OmegaConf
from copy import deepcopy

from ..utils.mics import fixseed
from ..config import Config
from ...utils import start_debug
from .train_platforms import *
from ..logger_new import mylogger
from ..utils.model_utils import create_model_and_diffusion
from .training_loop import TrainLoop
from .. import dist as dist_utils


@hydra.main(config_path=pjoin(os.getcwd(), 'conf'), config_name='config', version_base=None)
def main(cfg: Config):
    # start_debug()
    dist_utils.init_distributed_mode(backend='nccl')

    if cfg.train.resume_checkpoint != '':
        old_config_path = Path(cfg.train.resume_checkpoint).parent / "config.yaml"
        with open(old_config_path.as_posix(), "r") as f:
            old_config:Config = OmegaConf.load(f)

        if cfg.train.save_dir == old_config.train.save_dir:
            raise ValueError("Cannot resume from the same save_dir as the old config.")
        old_config.train.save_dir = cfg.train.save_dir
        old_config.train.resume_checkpoint = cfg.train.resume_checkpoint

        cfg = deepcopy(old_config)

    fixseed(cfg.seed)

    if dist_utils.is_main_process():

        if cfg.train.save_dir is None:
            raise FileNotFoundError("save_dir was not specified.")
        elif Path(cfg.train.save_dir).exists() and not cfg.train.overwrite:
            raise FileExistsError(f"save_dir {cfg.train.save_dir} already exists.")
        elif Path(cfg.train.save_dir).exists() and cfg.train.overwrite:
            while True:
                # user_input = input(f"Do you want to overwrite the existing exp dir {args.save_dir}? (yes/no): ")
                user_input = 'yes'
                if user_input.lower() == 'yes':
                    for entry in Path(cfg.train.save_dir).iterdir():
                        if entry.is_dir() and entry.name == 'hydra_log':
                            continue
                        if entry.is_file():
                            os.remove(entry.as_posix())
                        elif entry.is_dir():
                            shutil.rmtree(entry.as_posix())
                    break
                elif user_input.lower() == 'no':
                    sys.exit(0)

        elif not Path(cfg.train.save_dir).exists():
            Path(cfg.train.save_dir).mkdir(parents=True, exist_ok=True)


    dist_utils.barrier()
    if dist_utils.is_main_process():
        mylogger.setup(
            logger_name='MyLogger_MASTER',
            file_log_level=logging.DEBUG,
            stream_log_level=logging.DEBUG,
            log_file=(Path(cfg.train.save_dir) / 'train_master.log').as_posix(),
        )
    else:
        mylogger.setup(
            logger_name=f"MyLogger_RANK{dist_utils.get_rank()}",
            file_log_level=logging.INFO,
            stream_log_level=logging.INFO,
            log_file=(Path(cfg.train.save_dir) / f'train_rank{dist_utils.get_rank()}.log').as_posix(),
        )



    if dist_utils.is_main_process():
        args_path = (Path(cfg.train.save_dir) / 'config.yaml').as_posix()
        with open(args_path, 'w') as f:
            yaml.dump(OmegaConf.to_container(cfg, resolve=True), f)

    try:
        train_platform_type = eval(cfg.train.train_platform_type)
        train_platform = train_platform_type(cfg.train.save_dir)
        train_platform.report_args(cfg, name='Args')

        mylogger.info("Creating model and diffusion ...")
        model, diffusion = create_model_and_diffusion(cfg.model)
        mylogger.info(f"Model parameters: {sum(p.numel() for p in model.parameters()) / 1e6:.2f} M")

        # Print treble mask probability
        if hasattr(model, 'treble_mask_prob'):
            mylogger.info(f"Treble mask probability: {model.treble_mask_prob}")
            if model.treble_mask_prob == 1.0:
                mylogger.info("  -> No masking (keep all treble branches)")
            elif model.treble_mask_prob == 0.5:
                mylogger.info("  -> 50% masking (each branch has 50% chance to be masked)")
            else:
                mylogger.info(f"  -> {(1-model.treble_mask_prob)*100:.0f}% masking probability")

        mylogger.info("Training...")
        TrainLoop(cfg, train_platform, model, diffusion).run_loop()
        train_platform.close()

    except KeyboardInterrupt:
        print("\n[INFO] Training interrupted by user. Cleaning up ...")

    except Exception as e:
        mylogger.error(f"An error occurred during training: {e}")
        import traceback
        traceback.print_exc()

    finally:
        if dist_utils.is_dist_avail_and_initialized():
            dist_utils.cleanup()


if __name__ == "__main__":
    main()