diff --git a/main/app/app.py b/main/app/app.py deleted file mode 100644 index ad0ac945204e8dfe497599c3436fcd9ea8eaabec..0000000000000000000000000000000000000000 --- a/main/app/app.py +++ /dev/null @@ -1,190 +0,0 @@ -import os -import io -import ssl -import sys -import time -import codecs -import logging -import warnings - -import gradio as gr - -sys.path.append(os.getcwd()) -start_time = time.time() - -from main.app.variables import logger, config, translations, theme, font, configs, language, allow_disk, gradio_version - -from main.app.core.realtime import js_code -from main.app.tabs.extra.extra import extra_tab -from main.app.tabs.editing.editing import editing_tab -from main.app.tabs.training.training import training_tab -from main.app.tabs.realtime.realtime import realtime_tab -from main.app.tabs.downloads.downloads import download_tab -from main.app.tabs.inference.inference import inference_tab -from main.configs.rpc import connect_discord_ipc, send_discord_rpc - -ssl._create_default_https_context = ssl._create_unverified_context - -if not config.debug_mode: - warnings.filterwarnings("ignore") - for l in ["httpx", "gradio", "uvicorn", "httpcore", "urllib3"]: - logging.getLogger(l).setLevel(logging.ERROR) - -if sys.platform == "win32": - import asyncio.proactor_events as _pe - - _orig_ccl = _pe._ProactorBasePipeTransport._call_connection_lost - - def _ccl_patched(self, exc): - try: - _orig_ccl(self, exc) - except ConnectionResetError: - pass - - _pe._ProactorBasePipeTransport._call_connection_lost = _ccl_patched - -css = """ - -""".format( - font_urls=font or "https://fonts.googleapis.com/css2?family=Saira&display=swap", - fonts=( - font or "https://fonts.googleapis.com/css2?family=Saira&display=swap" - ).replace("https://fonts.googleapis.com/css2?family=", "").replace("+", " ").split(":")[0].split("&")[0] - -) - -client_mode = True # "--client" in sys.argv - -gr_params = { - "js": ( - ("() => {\n" + js_code + "\n}") - if gradio_version else - js_code - ) if client_mode else None, - "theme": theme, - "css": css -} - -with gr.Blocks( - title="📱 Vietnamese-RVC GUI", - **gr_params if gradio_version else {} -) as app: - gr.HTML("

🎵VIETNAMESE RVC🎵

", padding=True) - gr.HTML(f"

{translations['title']}

", padding=True) - - with gr.Tabs(): - if configs.get("inference_tab", True): inference_tab() - if configs.get("editing_tab", True): editing_tab() - if configs.get("realtime_tab", True) and not config.is_zluda: realtime_tab() - if configs.get("create_and_training_tab", True): training_tab() - if configs.get("downloads_tab", True): download_tab() - if configs.get("extra_tab", True): extra_tab(app) - - with gr.Row(): - gr.Markdown( - translations["rick_roll"].format( - rickroll=codecs.decode('uggcf://jjj.lbhghor.pbz/jngpu?i=qDj4j9JtKpD', 'rot13') - ) - ) - - with gr.Row(): - gr.Markdown(translations["terms_of_use"]) - - with gr.Row(): - gr.Markdown(translations["exemption"]) - - if __name__ == "__main__": - device = config.device.replace("privateuseone", "dml") - if config.is_zluda: device = device.replace("cuda", "zluda") - if config.hip_version is not None: device = device.replace("cuda", "hip") - - logger.info(f"Pytorch: {device} | Onnxruntime: {config.providers[0].replace('Dml', 'Ocl') if device.startswith('ocl') else config.providers[0]}") - if config.compile_all: logger.info(translations["compile_model"].format(compile_mode=config.compile_mode)) - logger.info(f'{translations["precision"]}: {("BF16" if config.brain else "FP16") if config.is_half else "FP32"}') - logger.info(translations["start_app"]) - logger.info(translations["set_lang"].format(lang=language)) - - port = configs.get("app_port", 7860) - server_name = configs.get("server_name", "0.0.0.0") - share = "--share" in sys.argv - - original_stdout = sys.stdout - sys.stdout = io.StringIO() - - for i in range(configs.get("num_of_restart", 5)): - try: - gradio_app, _, share_url = app.queue().launch( - favicon_path=configs["ico_path"], - server_name=server_name, - server_port=port, - show_error=configs.get("app_show_error", False), - inbrowser="--open" in sys.argv, - share=share, - allowed_paths=allow_disk, - prevent_thread_lock=True, - quiet=not config.debug_mode, - **gr_params if not gradio_version else {} - ) - break - except OSError: - logger.debug(translations["port"].format(port=port)) - port -= 1 - except Exception as e: - logger.error(translations["error_occurred"].format(e=e)) - sys.exit(1) - - if client_mode: - logger.debug(translations["mount_fastapi"]) - - from main.inference.realtime.client import app as fastapi_app - gradio_app.mount("/api", fastapi_app) - - logger.debug("\n" + sys.stdout.getvalue()) - sys.stdout = original_stdout - - if configs.get("discord_presence", True) and "--no_discord" not in sys.argv: - pipe = connect_discord_ipc() - if pipe: - try: - logger.info(translations["start_rpc"]) - send_discord_rpc(pipe) - except KeyboardInterrupt: - logger.info(translations["stop_rpc"]) - pipe.close() - - logger.info(f"{translations['running_local_url']}: {server_name}:{port}") - if share: logger.info(f"{translations['running_share_url']}: {share_url}") - - if "--tensorboard" in sys.argv: - logger.info(translations["run_tensorboard"]) - - from main.app.run_tensorboard import launch_tensorboard - launch_tensorboard() - - logger.info(f"{translations['gradio_start']}: {(time.time() - start_time):.2f}s") - - while 1: - time.sleep(5) \ No newline at end of file diff --git a/main/app/core/csrt.py b/main/app/core/csrt.py deleted file mode 100644 index 476532ac42f4d2ee40ac6eed010612f450a8c63d..0000000000000000000000000000000000000000 --- a/main/app/core/csrt.py +++ /dev/null @@ -1,122 +0,0 @@ -import os -import gc -import sys - -sys.path.append(os.getcwd()) - -from main.app.core.ui import gr_info, gr_warning, process_output -from main.app.variables import config, translations, configs, logger -from main.library.utils import check_spk_diarization, clear_gpu_cache - -def whisper_process( - model_size, - input_audio, - configs, - device, - out_queue, - language="vi" -): - from main.library.speaker_diarization.whisper import load_model - - try: - model = load_model( - model_size, - device=device - ) - - if config.compile_all: - import torch - - model.encoder = torch.compile(model.encoder, mode=config.compile_mode) - model.decoder = torch.compile(model.decoder, mode=config.compile_mode) - - segments = model.transcribe( - input_audio, - fp16=configs.get("fp16", False), - word_timestamps=True, - language=language - ) - - out_queue.put(segments["segments"]) - except Exception as e: - out_queue.put(e) - finally: - clear_gpu_cache() - del segments - gc.collect() - -def create_srt( - model_size, - input_audio, - output_file, - language -): - import multiprocessing as mp - - if not input_audio or not os.path.exists(input_audio) or os.path.isdir(input_audio): - gr_warning(translations["input_not_valid"]) - return [None]*2 - - if not output_file.endswith(".srt"): output_file += ".srt" - - if not output_file: - gr_warning(translations["output_not_valid"]) - return [None]*2 - - output_dir = os.path.dirname(output_file) - if not os.path.exists(output_dir): os.makedirs(output_dir, exist_ok=True) - - info = "" - output_file = process_output(output_file) - - check_spk_diarization(model_size, speechbrain=False) - gr_info(translations["csrt"]) - - ctx = mp.get_context("spawn") - whisper_queue = ctx.Queue() - - whisperprocess = ctx.Process( - target=whisper_process, - args=( - model_size, - input_audio, - configs, - config.device, - whisper_queue, - language - ) - ) - - whisperprocess.start() - segments = whisper_queue.get() - whisperprocess.join() - - with open(output_file, "w", encoding="utf-8") as f: - for i, segment in enumerate(segments): - start = segment["start"] - end = segment["end"] - text = segment["text"].strip() - - index = f"{i+1}\n" - timestamp = f"{format_timestamp(start)} --> {format_timestamp(end)}\n" - text1 = f"{text}\n\n" - - f.write(index) - f.write(timestamp) - f.write(text1) - - info = info + index + timestamp + text1 - logger.info(info) - - gr_info(translations["success"]) - - return [{"value": output_file, "visible": True, "__type__": "update"}, info] - -def format_timestamp(seconds): - hours = int(seconds // 3600) - minutes = int((seconds % 3600) // 60) - - seconds = int(seconds % 60) - miliseconds = int(round((seconds % 1) * 1000)) - - return f"{hours:02}:{minutes:02}:{seconds:02},{miliseconds:03}" \ No newline at end of file diff --git a/main/app/core/downloads.py b/main/app/core/downloads.py deleted file mode 100644 index 7813eef7c10507d92f28102616ad11c34bb1a11c..0000000000000000000000000000000000000000 --- a/main/app/core/downloads.py +++ /dev/null @@ -1,351 +0,0 @@ -import os -import re -import sys -import json -import codecs -import shutil -import yt_dlp -import warnings -import requests - -from bs4 import BeautifulSoup - -sys.path.append(os.getcwd()) - -from main.tools import huggingface, gdown, meganz, mediafire, pixeldrain -from main.app.variables import logger, translations, model_options, configs -from main.app.core.process import move_files_from_directory, fetch_pretrained_data, extract_name_model -from main.app.core.ui import gr_info, gr_warning, gr_error, process_output, replace_url, replace_modelname - -def download_url(url): - if not url: - gr_warning(translations["provide_url"]) - return [None]*3 - - if not os.path.exists(configs["audios_path"]): os.makedirs(configs["audios_path"], exist_ok=True) - - with warnings.catch_warnings(): - warnings.filterwarnings("ignore") - - ydl_opts = { - "format": "bestaudio/best", - "postprocessors": [{ - "key": "FFmpegExtractAudio", - "preferredcodec": "wav", - "preferredquality": "192" - }], - "quiet": True, - "no_warnings": True, - "noplaylist": True, - "verbose": False - } - - gr_info(translations["start"].format(start=translations["download_music"])) - audio_output = "" - - with yt_dlp.YoutubeDL(ydl_opts) as ydl: - audio_output = process_output(os.path.join( - configs["audios_path"], - re.sub( - r'\s+', '-', - re.sub( - r'[^\w\s\u4e00-\u9fff\uac00-\ud7af\u0400-\u04FF\u1100-\u11FF]', - '', - ydl.extract_info(url, download=False).get('title', 'video') - ).strip() - ) - ) + ".wav") - - if os.path.exists(audio_output): - shutil.rmtree(audio_output, ignore_errors=True) - - ydl_opts['outtmpl'] = audio_output.replace(".wav", "") - - with yt_dlp.YoutubeDL(ydl_opts) as ydl: - ydl.download([url]) - - gr_info(translations["success"]) - - return [ - audio_output, - audio_output, - translations["success"] - ] - -def move_file( - file, - download_dir, - model, - use_orig_weight_name=False -): - weights_dir = configs["weights_path"] - logs_dir = configs["logs_path"] - - if not os.path.exists(weights_dir): os.makedirs(weights_dir, exist_ok=True) - if not os.path.exists(logs_dir): os.makedirs(logs_dir, exist_ok=True) - - if file.endswith(".zip"): shutil.unpack_archive(file, download_dir) - move_files_from_directory(download_dir, weights_dir, logs_dir, model, use_orig_weight_name) - -def download_model(url=None, model=None): - if not url: return gr_warning(translations["provide_url"]) - logger.debug(model + ": " + url) - - url = replace_url(url) - download_dir = "download_model" - use_orig_weight_name = False - - os.makedirs(download_dir, exist_ok=True) - - try: - gr_info(translations["start"].format(start=translations["download"])) - - if "huggingface.co" in url: - file = huggingface.HF_download_file( - url, - download_dir - ) - elif "google.com" in url: - file = gdown.gdown_download( - url, - download_dir - ) - elif "mediafire.com" in url: - file = mediafire.Mediafire_Download( - url, - download_dir - ) - elif "pixeldrain.com" in url: - file = pixeldrain.pixeldrain( - url, - download_dir - ) - elif "mega.nz" in url: - file = meganz.mega_download_url( - url, - download_dir - ) - else: - gr_warning(translations["not_support_url"]) - return translations["not_support_url"] - - if not model: - use_orig_weight_name = True - modelname = os.path.basename(file) - - model = ( - extract_name_model(modelname) - if modelname.endswith(".index") else - os.path.splitext(modelname)[0] - ) - - if model is None: model = os.path.splitext(modelname)[0] - - model = replace_modelname(model) - move_file(file, download_dir, model, use_orig_weight_name) - - gr_info(translations["success"]) - return translations["success"] - except Exception as e: - gr_error(message=translations["error_occurred"].format(e=e)) - - return translations["error_occurred"].format(e=e) - finally: - shutil.rmtree(download_dir, ignore_errors=True) - -def download_pretrained_model( - choices, - model, - sample_rate -): - pretraineds_custom_path = configs["pretrained_custom_path"] - - if choices == translations["list_model"]: - paths = fetch_pretrained_data()[model][sample_rate] - if not os.path.exists(pretraineds_custom_path): - os.makedirs(pretraineds_custom_path, exist_ok=True) - - url = codecs.decode( - "uggcf://uhttvatsnpr.pb/NauC/Ivrganzrfr-EIP-Cebwrpg/erfbyir/znva/cergenvarq_phfgbz/", - "rot13" - ) + paths - - gr_info(translations["download_pretrain"]) - - file = huggingface.HF_download_file( - replace_url(url), - os.path.join(pretraineds_custom_path, paths) - ) - - if file.endswith(".zip"): - shutil.unpack_archive(file, pretraineds_custom_path) - os.remove(file) - - gr_info(translations["success"]) - return translations["success"] - elif choices == translations["download_url"]: - pretrain_is_zip = ( - model.endswith(".zip") or - model.endswith(".zip?download=true") or - sample_rate.endswith(".zip") or - sample_rate.endswith(".zip?download=true") - ) - - urls = [] - - if not model and not pretrain_is_zip: - gr_warning(translations["provide_pretrain"].format(dg="D")) - return [None]*2 - - if not sample_rate and not pretrain_is_zip: - gr_warning(translations["provide_pretrain"].format(dg="G")) - return [None]*2 - - gr_info(translations["download_pretrain"]) - - if model: urls.append(model) - if sample_rate: urls.append(sample_rate) - - for url in urls: - url = replace_url(url) - - if "huggingface.co" in url: - file = huggingface.HF_download_file( - url, - pretraineds_custom_path - ) - elif "google.com" in url: - file = gdown.gdown_download( - url, - pretraineds_custom_path - ) - elif "mediafire.com" in url: - file = mediafire.Mediafire_Download( - url, - pretraineds_custom_path - ) - elif "pixeldrain.com" in url: - file = pixeldrain.pixeldrain( - url, - pretraineds_custom_path - ) - elif "mega.nz" in url: - file = meganz.mega_download_url( - url, - pretraineds_custom_path - ) - else: - gr_warning(translations["not_support_url"]) - return translations["not_support_url"], translations["not_support_url"] - - if file.endswith(".zip"): - shutil.unpack_archive(file, pretraineds_custom_path) - if os.path.exists(file): os.remove(file) - - gr_info(translations["success"]) - return translations["success"], translations["success"] - -def is_empty_table(html): - return (' {str(datetime.timedelta(seconds=int(end_t)))}" - dur_str = f"{dur:.2f}s" - - rows.append([spk, time_range, dur_str]) - speaker_stats[spk] = speaker_stats.get(spk, 0.0) + dur - col_widths = [] - for i in range(3): - max_len = max(len(headers[i].replace("-", " ")), *(len(row[i]) for row in rows)) - col_widths.append(max_len) - - def make_line(left, mid, right): - return left + mid.join("─" * (w + 2) for w in col_widths) + right - - def format_row(row): - return "│ " + " │ ".join(f"{cell:<{col_widths[i]}}" for i, cell in enumerate(row)) + " │" - - lines.append("\n") - lines.append(make_line("┌", "┬", "┐")) - lines.append(format_row(headers)) - lines.append(make_line("├", "┼", "┤")) - - for row in rows: - lines.append(format_row(row)) - - lines.append(make_line("└", "┴", "┘")) - return "\n".join(lines) - - def merge_audio( - files_list, - time_stamps, - original_file_path, - output_path, - export_format - ): - def extract_number(filename): - match = re.search(r'_(\d+)', filename) - return int(match.group(1)) if match else 0 - - total_duration = len(pydub_load(original_file_path)) - combined = AudioSegment.empty() - current_position = 0 - - for file, (start_i, end_i) in zip(sorted(files_list, key=extract_number), time_stamps): - if start_i > current_position: - combined += AudioSegment.silent( - duration=start_i - current_position - ) - - combined += pydub_load(file) - current_position = end_i - - if current_position < total_duration: - combined += AudioSegment.silent( - duration=total_duration - current_position - ) - - combined.export(output_path, format=export_format) - return output_path - - embeddings = np.zeros(shape=(len(segments), 192)) - for i, segment in enumerate(segments): - embeddings[i] = segment_embedding(segment) - - labels = AgglomerativeClustering(num_spk).fit(np.nan_to_num(embeddings)).labels_ - for i in range(len(segments)): - segments[i]["speaker"] = f"SPEAKER {labels[i] + 1}" - - merged_segments = [] - - if segments: - curr = segments[0].copy() - - for i in range(1, len(segments)): - if segments[i]["speaker"] == curr["speaker"]: curr["end"] = segments[i]["end"] - else: - merged_segments.append(curr) - curr = segments[i].copy() - - merged_segments.append(curr) - - gr_info(translations["analysis_completed"].format(length=len(merged_segments))) - - del audio, embedding_model, segments, labels - clear_gpu_cache() - gc.collect() - - gr_info(translations["process_audio"]) - - pydub_audio = pydub_load(input_audio) - output_folder = "audios_temp" - - if os.path.exists(output_folder): shutil.rmtree(output_folder, ignore_errors=True) - - for f in [ - output_folder, - os.path.join(output_folder, "1"), - os.path.join(output_folder, "2") - ]: - os.makedirs(f, exist_ok=True) - - time_stamps, processed_segments = [], [] - for i, seg in enumerate(merged_segments): - model_id = 1 if int(seg["speaker"].split()[-1]) % 2 != 0 else 2 - start_ms, end_ms = int(seg["start"] * 1000), int(seg["end"] * 1000) - - target_path = os.path.join(output_folder, str(model_id), f"segment_{i+1}.wav") - pydub_audio[start_ms:end_ms].export(target_path, format="wav") - - processed_segments.append(os.path.join(output_folder, str(model_id), f"segment_{i+1}_output.wav")) - time_stamps.append((start_ms, end_ms)) - - logger.info(get_formatted_stats(merged_segments)) - - f0method, embedder_model = ( - f0_method if f0_method != "hybrid" else hybrid_method, - embedders if embedders != "custom" else custom_embedders - ) - - gr_info(translations["process_done_start_convert"]) - - convert( - pitch_1, - filter_radius, - index_strength_1, - rms_mix_rate, - protect, - hop_length, - f0method, - os.path.join(output_folder, "1"), - output_folder, - model_pth_1, - model_index_1, - autotune, - cleaner, - clean_strength, - "wav", - embedder_model, - 48000 if resample_sr == 0 else resample_sr, - False, - f0_autotune_strength, - checkpointing, - predictor_onnx, - embed_mode, - formant_shifting, - formant_qfrency_1, - formant_timbre_1, - "", - proposal_pitch, - proposal_pitch_threshold, - audio_processing, - alpha, - sid_1, - embedders_mix, - embedders_mix_layers, - embedders_mix_ratio, - noise_scale_1, - nprobe_1, - False - ) - - convert( - pitch_2, - filter_radius, - index_strength_2, - rms_mix_rate, - protect, - hop_length, - f0method, - os.path.join(output_folder, "2"), - output_folder, - model_pth_2, - model_index_2, - autotune, - cleaner, - clean_strength, - "wav", - embedder_model, - 48000 if resample_sr == 0 else resample_sr, - False, - f0_autotune_strength, - checkpointing, - predictor_onnx, - embed_mode, - formant_shifting, - formant_qfrency_2, - formant_timbre_2, - "", - proposal_pitch, - proposal_pitch_threshold, - audio_processing, - alpha, - sid_2, - embedders_mix, - embedders_mix_layers, - embedders_mix_ratio, - noise_scale_2, - nprobe_2, - False - ) - - gr_info(translations["convert_success"]) - - return merge_audio( - processed_segments, - time_stamps, - input_audio, - replace_export_format(output_audio, export_format), - export_format - ) - except Exception as e: - import traceback - - gr_error(translations["error_occurred"].format(e=e)) - logger.debug(traceback.format_exc()) - - return None - finally: - if os.path.exists("audios_temp"): - shutil.rmtree("audios_temp", ignore_errors=True) - -def convert_tts( - clean_audio, - autotune, - pitch, - clean_strength, - model, - index, - index_rate, - input_path, - output_path, - export_format, - method, - hybrid_method, - hop_length, - embedders, - custom_embedders, - resample_sr, - filter_radius, - rms_mix_rate, - protect, - split_audio, - f0_autotune_strength, - checkpointing, - predictor_onnx, - formant_shifting, - formant_qfrency, - formant_timbre, - f0_file, - embedders_mode, - proposal_pitch, - proposal_pitch_threshold, - audio_processing=False, - alpha=0.5, - sid=0, - embedders_mix = False, - embedders_mix_layers = 9, - embedders_mix_ratio = 0.5, - noise_scale = 0.35, - nprobe = 1, - audio_upscaler = False -): - model_path = os.path.join(configs["weights_path"], model) if not os.path.exists(model) else model - - if ( - not model_path or - not os.path.exists(model_path) or - os.path.isdir(model_path) or - not model.endswith((".pth", ".onnx")) - ): - gr_warning(translations["provide_file"].format(filename=translations["model"])) - return None - - if not input_path or not os.path.exists(input_path): - gr_warning(translations["input_not_valid"]) - return None - - if os.path.isdir(input_path): - input_audio = [ - f for f in os.listdir(input_path) - if "tts" in f and f.lower().endswith(tuple(file_types)) - ] - - if not input_audio: - gr_warning(translations["not_found_in_folder"]) - return None - - input_path = os.path.join(input_path, input_audio[0]) - - if not output_path: - gr_warning(translations["output_not_valid"]) - return None - - output_path = replace_export_format(output_path, export_format) - if os.path.isdir(output_path): output_path = os.path.join(output_path, f"tts.{export_format}") - - output_dir = os.path.dirname(output_path) - if not os.path.exists(output_dir): os.makedirs(output_dir, exist_ok=True) - - output_path = process_output(output_path) - - f0method, embedder_model = ( - method if method != "hybrid" else hybrid_method, - embedders if embedders != "custom" else custom_embedders - ) - - gr_info(translations["convert_vocal"]) - - convert( - pitch, - filter_radius, - index_rate, - rms_mix_rate, - protect, - hop_length, - f0method, - input_path, - output_path, - model_path, - index, - autotune, - clean_audio, - clean_strength, - export_format, - embedder_model, - resample_sr, - split_audio, - f0_autotune_strength, - checkpointing, - predictor_onnx, - embedders_mode, - formant_shifting, - formant_qfrency, - formant_timbre, - f0_file, - proposal_pitch, - proposal_pitch_threshold, - audio_processing, - alpha, - sid, - embedders_mix, - embedders_mix_layers, - embedders_mix_ratio, - noise_scale, - nprobe, - audio_upscaler - ) - - gr_info(translations["convert_success"]) - return output_path \ No newline at end of file diff --git a/main/app/core/model_utils.py b/main/app/core/model_utils.py deleted file mode 100644 index 56e6eb9100f240a2d7332d0043325dbbd7900598..0000000000000000000000000000000000000000 --- a/main/app/core/model_utils.py +++ /dev/null @@ -1,301 +0,0 @@ -import os -import sys -import json -import torch -import datetime - -sys.path.append(os.getcwd()) - -from main.app.core.ui import gr_info, gr_warning, gr_error -from main.app.variables import config, logger, translations, configs - -def fushion_model_pth( - name, - model_path_1, - model_path_2, - ratio -): - if not name.endswith(".pth"): name = name + ".pth" - - model_path_1 = os.path.join(configs["weights_path"], model_path_1) if not os.path.exists(model_path_1) else model_path_1 - model_path_2 = os.path.join(configs["weights_path"], model_path_2) if not os.path.exists(model_path_2) else model_path_2 - - if ( - not model_path_1 or - not os.path.exists(model_path_1) or - not model_path_1.endswith(".pth") - ): - gr_warning(translations["provide_file"].format(filename=translations["model"] + " 1")) - return [translations["provide_file"].format(filename=translations["model"] + " 1"), None] - - if ( - not model_path_2 or - not os.path.exists(model_path_2) or - not model_path_2.endswith(".pth") - ): - gr_warning(translations["provide_file"].format(filename=translations["model"] + " 2")) - return [translations["provide_file"].format(filename=translations["model"] + " 2"), None] - - from collections import OrderedDict - - def extract(ckpt): - a = ckpt["model"] - opt = OrderedDict() - opt["weight"] = {} - - for key in a.keys(): - if "enc_q" in key: continue - - opt["weight"][key] = a[key] - - return opt - - try: - ckpt1 = torch.load(model_path_1, map_location="cpu", weights_only=True) - ckpt2 = torch.load(model_path_2, map_location="cpu", weights_only=True) - - if ckpt1["sr"] != ckpt2["sr"]: - gr_warning(translations["sr_not_same"]) - return [translations["sr_not_same"], None] - - cfg = ckpt1["config"] - cfg_f0 = ckpt1["f0"] - cfg_version = ckpt1["version"] - cfg_sr = ckpt1["sr"] - - vocoder = ckpt1.get("vocoder", "Default") - rms_extract = ckpt1.get("energy", False) - - ckpt1 = extract(ckpt1) if "model" in ckpt1 else ckpt1["weight"] - ckpt2 = extract(ckpt2) if "model" in ckpt2 else ckpt2["weight"] - - if sorted(list(ckpt1.keys())) != sorted(list(ckpt2.keys())): - gr_warning(translations["architectures_not_same"]) - - return [translations["architectures_not_same"], None] - - gr_info(translations["start"].format(start=translations["fushion_model"])) - - opt = OrderedDict() - opt["weight"] = {} - - for key in ckpt1.keys(): - if key == "emb_g.weight" and ckpt1[key].shape != ckpt2[key].shape: - min_shape0 = min(ckpt1[key].shape[0], ckpt2[key].shape[0]) - - opt["weight"][key] = ( - ratio * (ckpt1[key][:min_shape0].float()) + (1 - ratio) * (ckpt2[key][:min_shape0].float()) - ).half() - else: - opt["weight"][key] = ( - ratio * (ckpt1[key].float()) + (1 - ratio) * (ckpt2[key].float()) - ).half() - - opt["config"] = cfg - opt["sr"] = cfg_sr - opt["f0"] = cfg_f0 - opt["version"] = cfg_version - - opt["infos"] = translations["model_fushion_info"].format( - name=name, - model_path_1=model_path_1, - model_path_2=model_path_2, - ratio=ratio - ) - - opt["vocoder"] = vocoder - opt["energy"] = rms_extract - - output_model = configs["weights_path"] - if not os.path.exists(output_model): os.makedirs(output_model, exist_ok=True) - - torch.save(opt, os.path.join(output_model, name)) - - gr_info(translations["success"]) - return [translations["success"], os.path.join(output_model, name)] - except Exception as e: - gr_error(message=translations["error_occurred"].format(e=e)) - return [e, None] - -def fushion_model( - modelname, - model_path_1, - model_path_2, - ratio -): - if not modelname: - gr_warning(translations["provide_name_is_save"]) - return [translations["provide_name_is_save"], None] - - if model_path_1.endswith(".pth") and model_path_2.endswith(".pth"): - return fushion_model_pth( - modelname, - model_path_1, - model_path_2, - ratio - ) - else: - gr_warning(translations["format_not_valid"]) - return [None, None] - -def onnx_export(model_path): - if not model_path.endswith(".pth"): model_path += ".pth" - model_path = os.path.join(configs["weights_path"], model_path) if not os.path.exists(model_path) else model_path - - if ( - not model_path or - not os.path.exists(model_path) or - not model_path.endswith(".pth") - ): - return gr_warning(translations["provide_file"].format(filename=translations["model"])) - - try: - gr_info(translations["start_onnx_export"]) - - from main.library.onnx.onnx_export import onnx_exporter - - output = onnx_exporter( - model_path, - model_path.replace(".pth", ".onnx"), - is_half=config.is_half, - device=config.device - ) - - gr_info(translations["success"]) - return output - except Exception as e: - return gr_error(e) - -def svc_export(model_path, config_path, modelname, delete_when_success): - if not model_path.endswith(".pth"): model_path += ".pth" - model_path = os.path.join(configs["weights_path"], model_path) if not os.path.exists(model_path) else model_path - - if ( - not model_path or - not os.path.exists(model_path) or - not model_path.endswith(".pth") - ): - gr_warning(translations["provide_file"].format(filename=translations["model"])) - return None - - if ( - not config_path or - not os.path.exists(config_path) or - not config_path.endswith(".json") - ): - gr_warning(translations["provide_file"].format(filename=translations["config"])) - return None - - if modelname is None: modelname = os.path.basename(model_path) - if not modelname.endswith(".pth"): modelname += ".pth" - output_path = os.path.join(configs["weights_path"], modelname) if not os.path.exists(modelname) else modelname - - if ( - not model_path or - not os.path.exists(model_path) or - not model_path.endswith(".pth") - ): - return gr_warning(translations["provide_file"].format(filename=translations["model"])) - - try: - gr_info(translations["start_convert_svc"]) - - from main.tools.export_svc import svc_converter - - output = svc_converter( - model_path, - config_path, - output_path, - is_half=config.is_half - ) - - if output == translations["not_svc"]: - gr_warning(output) - return None - - if delete_when_success: os.remove(model_path); os.remove(config_path) - gr_info(translations["success"]) - return output - except Exception as e: - return gr_error(e) - -def model_info(model_path): - model_path = os.path.join(configs["weights_path"], model_path) if not os.path.exists(model_path) else model_path - - if ( - not model_path or - not os.path.exists(model_path) or - os.path.isdir(model_path) or - not model_path.endswith((".pth", ".onnx")) - ): - return gr_warning(translations["provide_file"].format(filename=translations["model"])) - - def prettify_date(date_str): - if date_str == translations["not_found_create_time"]: return None - - try: - return datetime.datetime.strptime( - date_str, - "%Y-%m-%dT%H:%M:%S.%f" - ).strftime("%Y-%m-%d %H:%M:%S") - except ValueError as e: - logger.debug(e) - return translations["format_not_valid"] - - if model_path.endswith(".pth"): - model_data = torch.load(model_path, map_location="cpu") - else: - import onnx - - model = onnx.load(model_path) - model_data = None - - for prop in model.metadata_props: - if prop.key == "model_info": - model_data = json.loads(prop.value) - break - - gr_info(translations["read_info"]) - - epochs = model_data.get("epoch", None) - if not epochs: - epochs = model_data.get("info", None) - - try: - epochs = int(epochs.split("e")[0].isdigit()) - if not epochs: epochs = translations["not_found"].format(name=translations["epoch"]) - except: - epochs = translations["not_found"].format(name=translations["epoch"]) - - steps = model_data.get("step", translations["not_found"].format(name=translations["step"])) - sr = model_data.get("sr", translations["not_found"].format(name=translations["sr"])) - f0 = model_data.get("f0", translations["not_found"].format(name=translations["f0"])) - version = model_data.get("version", translations["not_found"].format(name=translations["version"])) - creation_date = model_data.get("creation_date", translations["not_found_create_time"]) - model_hash = model_data.get("model_hash", translations["not_found"].format(name="model_hash")) - pitch_guidance = translations["trained_f0"] if f0 else translations["not_f0"] - creation_date_str = prettify_date(creation_date) if creation_date else translations["not_found_create_time"] - model_name = model_data.get("model_name", translations["unregistered"]) - model_author = model_data.get("author", translations["not_author"]) - vocoder = model_data.get("vocoder", "Default") - rms_extract = model_data.get("energy", False) - speakers_id = model_data.get("speakers_id", 1) - architecture = model_data.get("architecture", "RVC") - - gr_info(translations["success"]) - - return translations["model_info"].format( - model_name=model_name, - model_author=model_author, - epochs=epochs, - steps=steps, - version=version, - sr=sr, - pitch_guidance=pitch_guidance, - model_hash=model_hash, - creation_date_str=creation_date_str, - vocoder=vocoder, - rms_extract=rms_extract, - speakers_id=speakers_id, - architecture=architecture - ) \ No newline at end of file diff --git a/main/app/core/presets.py b/main/app/core/presets.py deleted file mode 100644 index 05b865db02c1669b00d3c64d0a1f899963795a4c..0000000000000000000000000000000000000000 --- a/main/app/core/presets.py +++ /dev/null @@ -1,744 +0,0 @@ -import os -import sys -import json - -sys.path.append(os.getcwd()) - -from main.app.variables import translations, configs -from main.app.core.ui import gr_info, gr_warning, change_preset_choices, change_effect_preset_choices, change_realtime_preset_choices - -def load_presets( - presets, - cleaner, - autotune, - pitch, - clean_strength, - index_strength, - resample_sr, - filter_radius, - rms_mix_rate, - protect, - split_audio, - f0_autotune_strength, - formant_shifting, - formant_qfrency, - formant_timbre, - proposal_pitch, - proposal_pitch_threshold, - embedders_mix, - embedders_mix_layers, - embedders_mix_ratio -): - if not presets: gr_warning(translations["provide_file_settings"]) - - file = {} - if presets: - with open(os.path.join(configs["presets_path"], presets)) as f: - file = json.load(f) - - gr_info(translations["load_presets"].format(presets=presets)) - - return [ - file.get("cleaner", cleaner), - file.get("autotune", autotune), - file.get("pitch", pitch), - file.get("clean_strength", clean_strength), - file.get("index_strength", index_strength), - file.get("resample_sr", resample_sr), - file.get("filter_radius", filter_radius), - file.get("rms_mix_rate", rms_mix_rate), - file.get("protect", protect), - file.get("split_audio", split_audio), - file.get("f0_autotune_strength", f0_autotune_strength), - file.get("formant_shifting", formant_shifting), - file.get("formant_qfrency", formant_qfrency), - file.get("formant_timbre", formant_timbre), - file.get("proposal_pitch", proposal_pitch), - file.get("proposal_pitch_threshold", proposal_pitch_threshold), - file.get("embedders_mix", embedders_mix), - file.get("embedders_mix_layers", embedders_mix_layers), - file.get("embedders_mix_ratio", embedders_mix_ratio) - ] - -def save_presets( - name, - cleaner, - autotune, - pitch, - clean_strength, - index_strength, - resample_sr, - filter_radius, - rms_mix_rate, - protect, - split_audio, - f0_autotune_strength, - cleaner_checkbox, - autotune_checkbox, - pitch_checkbox, - index_strength_checkbox, - resample_sr_checkbox, - filter_radius_checkbox, - rms_mix_rate_checkbox, - protect_checkbox, - split_audio_checkbox, - formant_shifting_checkbox, - formant_shifting, - formant_qfrency, - formant_timbre, - proposal_pitch, - proposal_pitch_threshold, - embedders_mix, - embedders_mix_layers, - embedders_mix_ratio -): - if not name: return gr_warning(translations["provide_filename_settings"]) - - if not any([ - cleaner_checkbox, - autotune_checkbox, - pitch_checkbox, - index_strength_checkbox, - resample_sr_checkbox, - filter_radius_checkbox, - rms_mix_rate_checkbox, - protect_checkbox, - split_audio_checkbox, - formant_shifting_checkbox - ]): - return gr_warning(translations["choose1"]) - - settings = {} - - for checkbox, data in [ - ( - cleaner_checkbox, - {"cleaner": cleaner, "clean_strength": clean_strength} - ), - ( - autotune_checkbox, - {"autotune": autotune, "f0_autotune_strength": f0_autotune_strength}), - ( - pitch_checkbox, - {"pitch": pitch} - ), - ( - index_strength_checkbox, - {"index_strength": index_strength} - ), - ( - resample_sr_checkbox, - {"resample_sr": resample_sr} - ), - ( - filter_radius_checkbox, - {"filter_radius": filter_radius} - ), - ( - rms_mix_rate_checkbox, - {"rms_mix_rate": rms_mix_rate} - ), - ( - protect_checkbox, - {"protect": protect} - ), - ( - split_audio_checkbox, - {"split_audio": split_audio} - ), - ( - formant_shifting_checkbox, - {"formant_shifting": formant_shifting, "formant_qfrency": formant_qfrency, "formant_timbre": formant_timbre} - ), - ( - proposal_pitch, - {"proposal_pitch": proposal_pitch, "proposal_pitch_threshold": proposal_pitch_threshold} - ), - ( - embedders_mix, - {"embedders_mix_layers": embedders_mix_layers, "embedders_mix_ratio": embedders_mix_ratio} - ) - ]: - if checkbox: settings.update(data) - - with open(os.path.join(configs["presets_path"], name + ".conversion.json"), "w") as f: - json.dump(settings, f, indent=4) - - gr_info(translations["export_settings"].format(name=name)) - return change_preset_choices() - -def audio_effect_load_presets( - presets, - resample_checkbox, - audio_effect_resample_sr, - chorus_depth, - chorus_rate_hz, - chorus_mix, - chorus_center_delay_ms, - chorus_feedback, - distortion_drive_db, - reverb_room_size, - reverb_damping, - reverb_wet_level, - reverb_dry_level, - reverb_width, - reverb_freeze_mode, - pitch_shift_semitones, - delay_second, - delay_feedback, - delay_mix, - compressor_threshold_db, - compressor_ratio, - compressor_attack_ms, - compressor_release_ms, - limiter_threshold_db, - limiter_release_ms, - gain_db, - bitcrush_bit_depth, - clipping_threshold_db, - phaser_rate_hz, - phaser_depth, - phaser_centre_frequency_hz, - phaser_feedback, - phaser_mix, - bass_boost, - bass_frequency, - treble_boost, - treble_frequency, - fade_in, - fade_out, - chorus_check_box, - distortion_checkbox, - reverb_check_box, - delay_check_box, - compressor_check_box, - limiter, - gain_checkbox, - bitcrush_checkbox, - clipping_checkbox, - phaser_check_box, - bass_or_treble, - fade -): - if not presets: gr_warning(translations["provide_file_settings"]) - - file = {} - if presets: - with open(os.path.join(configs["presets_path"], presets)) as f: - file = json.load(f) - - gr_info(translations["load_presets"].format(presets=presets)) - - return [ - file.get("resample_checkbox", resample_checkbox), - file.get("audio_effect_resample_sr", audio_effect_resample_sr), - file.get("chorus_depth", chorus_depth), - file.get("chorus_rate_hz", chorus_rate_hz), - file.get("chorus_mix", chorus_mix), - file.get("chorus_center_delay_ms", chorus_center_delay_ms), - file.get("chorus_feedback", chorus_feedback), - file.get("distortion_drive_db", distortion_drive_db), - file.get("reverb_room_size", reverb_room_size), - file.get("reverb_damping", reverb_damping), - file.get("reverb_wet_level", reverb_wet_level), - file.get("reverb_dry_level", reverb_dry_level), - file.get("reverb_width", reverb_width), - file.get("reverb_freeze_mode", reverb_freeze_mode), - file.get("pitch_shift_semitones", pitch_shift_semitones), - file.get("delay_second", delay_second), - file.get("delay_feedback", delay_feedback), - file.get("delay_mix", delay_mix), - file.get("compressor_threshold_db", compressor_threshold_db), - file.get("compressor_ratio", compressor_ratio), - file.get("compressor_attack_ms", compressor_attack_ms), - file.get("compressor_release_ms", compressor_release_ms), - file.get("limiter_threshold_db", limiter_threshold_db), - file.get("limiter_release_ms", limiter_release_ms), - file.get("gain_db", gain_db), - file.get("bitcrush_bit_depth", bitcrush_bit_depth), - file.get("clipping_threshold_db", clipping_threshold_db), - file.get("phaser_rate_hz", phaser_rate_hz), - file.get("phaser_depth", phaser_depth), - file.get("phaser_centre_frequency_hz", phaser_centre_frequency_hz), - file.get("phaser_feedback", phaser_feedback), - file.get("phaser_mix", phaser_mix), - file.get("bass_boost", bass_boost), - file.get("bass_frequency", bass_frequency), - file.get("treble_boost", treble_boost), - file.get("treble_frequency", treble_frequency), - file.get("fade_in", fade_in), - file.get("fade_out", fade_out), - file.get("chorus_check_box", chorus_check_box), - file.get("distortion_checkbox", distortion_checkbox), - file.get("reverb_check_box", reverb_check_box), - file.get("delay_check_box", delay_check_box), - file.get("compressor_check_box", compressor_check_box), - file.get("limiter", limiter), - file.get("gain_checkbox", gain_checkbox), - file.get("bitcrush_checkbox", bitcrush_checkbox), - file.get("clipping_checkbox", clipping_checkbox), - file.get("phaser_check_box", phaser_check_box), - file.get("bass_or_treble", bass_or_treble), - file.get("fade", fade) - ] - -def audio_effect_save_presets( - name, - resample_checkbox, - audio_effect_resample_sr, - chorus_depth, - chorus_rate_hz, - chorus_mix, - chorus_center_delay_ms, - chorus_feedback, - distortion_drive_db, - reverb_room_size, - reverb_damping, - reverb_wet_level, - reverb_dry_level, - reverb_width, - reverb_freeze_mode, - pitch_shift_semitones, - delay_second, - delay_feedback, - delay_mix, - compressor_threshold_db, - compressor_ratio, - compressor_attack_ms, - compressor_release_ms, - limiter_threshold_db, - limiter_release_ms, - gain_db, - bitcrush_bit_depth, - clipping_threshold_db, - phaser_rate_hz, - phaser_depth, - phaser_centre_frequency_hz, - phaser_feedback, - phaser_mix, - bass_boost, - bass_frequency, - treble_boost, - treble_frequency, - fade_in, - fade_out, - chorus_check_box, - distortion_checkbox, - reverb_check_box, - delay_check_box, - compressor_check_box, - limiter, - gain_checkbox, - bitcrush_checkbox, - clipping_checkbox, - phaser_check_box, - bass_or_treble, - fade -): - if not name: return gr_warning(translations["provide_filename_settings"]) - - if not any([ - resample_checkbox, - chorus_check_box, - distortion_checkbox, - reverb_check_box, - delay_check_box, - compressor_check_box, - limiter, - gain_checkbox, - bitcrush_checkbox, - clipping_checkbox, - phaser_check_box, - bass_or_treble, - fade, - pitch_shift_semitones != 0 - ]): - return gr_warning(translations["choose1"]) - - settings = {} - - for checkbox, data in [ - (resample_checkbox, { - "resample_checkbox": resample_checkbox, - "audio_effect_resample_sr": audio_effect_resample_sr - }), - (chorus_check_box, { - "chorus_check_box": chorus_check_box, - "chorus_depth": chorus_depth, - "chorus_rate_hz": chorus_rate_hz, - "chorus_mix": chorus_mix, - "chorus_center_delay_ms": chorus_center_delay_ms, - "chorus_feedback": chorus_feedback - }), - (distortion_checkbox, { - "distortion_checkbox": distortion_checkbox, - "distortion_drive_db": distortion_drive_db - }), - (reverb_check_box, { - "reverb_check_box": reverb_check_box, - "reverb_room_size": reverb_room_size, - "reverb_damping": reverb_damping, - "reverb_wet_level": reverb_wet_level, - "reverb_dry_level": reverb_dry_level, - "reverb_width": reverb_width, - "reverb_freeze_mode": reverb_freeze_mode - }), - (pitch_shift_semitones != 0, { - "pitch_shift_semitones": pitch_shift_semitones - }), - (delay_check_box, { - "delay_check_box": delay_check_box, - "delay_second": delay_second, - "delay_feedback": delay_feedback, - "delay_mix": delay_mix - }), - (compressor_check_box, { - "compressor_check_box": compressor_check_box, - "compressor_threshold_db": compressor_threshold_db, - "compressor_ratio": compressor_ratio, - "compressor_attack_ms": compressor_attack_ms, - "compressor_release_ms": compressor_release_ms - }), - (limiter, { - "limiter": limiter, - "limiter_threshold_db": limiter_threshold_db, - "limiter_release_ms": limiter_release_ms - }), - (gain_checkbox, { - "gain_checkbox": gain_checkbox, - "gain_db": gain_db - }), - (bitcrush_checkbox, { - "bitcrush_checkbox": bitcrush_checkbox, - "bitcrush_bit_depth": bitcrush_bit_depth - }), - (clipping_checkbox, { - "clipping_checkbox": clipping_checkbox, - "clipping_threshold_db": clipping_threshold_db - }), - (phaser_check_box, { - "phaser_check_box": phaser_check_box, - "phaser_rate_hz": phaser_rate_hz, - "phaser_depth": phaser_depth, - "phaser_centre_frequency_hz": phaser_centre_frequency_hz, - "phaser_feedback": phaser_feedback, - "phaser_mix": phaser_mix - }), - (bass_or_treble, { - "bass_or_treble": bass_or_treble, - "bass_boost": bass_boost, - "bass_frequency": bass_frequency, - "treble_boost": treble_boost, - "treble_frequency": treble_frequency - }), - (fade, { - "fade": fade, - "fade_in": fade_in, - "fade_out": fade_out - }) - ]: - if checkbox: settings.update(data) - - with open(os.path.join(configs["presets_path"], name + ".effect.json"), "w") as f: - json.dump(settings, f, indent=4) - - gr_info(translations["export_settings"].format(name=name)) - return change_effect_preset_choices() - -def save_realtime_config( - name, - exclusive_mode, - vad_enabled, - chunk_size, - pitch, - index_strength, - predictor_onnx, - hop_length, - embedder_mode, - f0_autotune, - proposal_pitch, - f0_autotune_strength, - proposal_pitch_threshold, - rms_mix_rate, - protect, - filter_radius, - silent_threshold, - extra_convert_size, - cross_fade_overlap_size, - vad_sensitivity, - vad_frame_ms, - clean_audio, - clean_strength, - post_process, - embedders_mix, - embedders_mix_layers, - embedders_mix_ratio, - chorus, - distortion, - reverb, - pitch_shift, - delay, - compressor, - limiter, - gain, - bitcrush, - clipping, - phaser, - chorus_depth, - chorus_rate, - chorus_mix, - chorus_center_delay, - chorus_feedback, - distortion_gain, - reverb_room_size, - reverb_damping, - reverb_wet_gain, - reverb_dry_gain, - reverb_width, - reverb_freeze_mode, - pitch_shift_semitones, - delay_seconds, - delay_feedback, - delay_mix, - compressor_threshold, - compressor_ratio, - compressor_attack, - compressor_release, - limiter_threshold, - limiter_release_time, - gain_db, - bitcrush_bit_depth, - clipping_threshold, - phaser_rate_hz, - phaser_depth, - phaser_centre_frequency_hz, - phaser_feedback, - phaser_mix -): - if not name: return gr_warning(translations["provide_filename_settings"]) - - settings = { - "exclusive_mode": exclusive_mode, - "vad_enabled": vad_enabled, - "chunk_size": chunk_size, - "pitch": pitch, - "index_strength": index_strength, - "predictor_onnx": predictor_onnx, - "hop_length": hop_length, - "embedder_mode": embedder_mode, - "embedders_mix": embedders_mix, - "embedders_mix_layers": embedders_mix_layers, - "embedders_mix_ratio": embedders_mix_ratio, - "f0_autotune": f0_autotune, - "proposal_pitch": proposal_pitch, - "f0_autotune_strength": f0_autotune_strength, - "proposal_pitch_threshold": proposal_pitch_threshold, - "rms_mix_rate": rms_mix_rate, - "protect": protect, - "filter_radius": filter_radius, - "silent_threshold": silent_threshold, - "extra_convert_size": extra_convert_size, - "cross_fade_overlap_size": cross_fade_overlap_size, - "vad_sensitivity": vad_sensitivity, - "vad_frame_ms": vad_frame_ms, - "clean_audio": clean_audio, - "clean_strength": clean_strength, - "post_process": post_process, - "chorus": chorus, - "distortion": distortion, - "reverb": reverb, - "pitch_shift": pitch_shift, - "delay": delay, - "compressor": compressor, - "limiter": limiter, - "gain": gain, - "bitcrush": bitcrush, - "clipping": clipping, - "phaser": phaser, - "chorus_depth": chorus_depth, - "chorus_rate": chorus_rate, - "chorus_mix": chorus_mix, - "chorus_center_delay": chorus_center_delay, - "chorus_feedback": chorus_feedback, - "distortion_gain": distortion_gain, - "reverb_room_size": reverb_room_size, - "reverb_damping": reverb_damping, - "reverb_wet_gain": reverb_wet_gain, - "reverb_dry_gain": reverb_dry_gain, - "reverb_width": reverb_width, - "reverb_freeze_mode": reverb_freeze_mode, - "pitch_shift_semitones": pitch_shift_semitones, - "delay_seconds": delay_seconds, - "delay_feedback": delay_feedback, - "delay_mix": delay_mix, - "compressor_threshold": compressor_threshold, - "compressor_ratio": compressor_ratio, - "compressor_attack": compressor_attack, - "compressor_release": compressor_release, - "limiter_threshold": limiter_threshold, - "limiter_release_time": limiter_release_time, - "gain_db": gain_db, - "bitcrush_bit_depth": bitcrush_bit_depth, - "clipping_threshold": clipping_threshold, - "phaser_rate_hz": phaser_rate_hz, - "phaser_depth": phaser_depth, - "phaser_centre_frequency_hz": phaser_centre_frequency_hz, - "phaser_feedback": phaser_feedback, - "phaser_mix": phaser_mix - } - - path = os.path.join(configs["presets_path"], name + ".realtime.json") - with open(path, "w") as f: - json.dump(settings, f, indent=4) - - gr_info(translations["export_settings"].format(name=name)) - return change_realtime_preset_choices() - -def load_realtime_config( - presets, - exclusive_mode, - vad_enabled, - chunk_size, - pitch, - index_strength, - predictor_onnx, - hop_length, - embedder_mode, - f0_autotune, - proposal_pitch, - f0_autotune_strength, - proposal_pitch_threshold, - rms_mix_rate, - protect, - filter_radius, - silent_threshold, - extra_convert_size, - cross_fade_overlap_size, - vad_sensitivity, - vad_frame_ms, - clean_audio, - clean_strength, - post_process, - embedders_mix, - embedders_mix_layers, - embedders_mix_ratio, - chorus, - distortion, - reverb, - pitch_shift, - delay, - compressor, - limiter, - gain, - bitcrush, - clipping, - phaser, - chorus_depth, - chorus_rate, - chorus_mix, - chorus_center_delay, - chorus_feedback, - distortion_gain, - reverb_room_size, - reverb_damping, - reverb_wet_gain, - reverb_dry_gain, - reverb_width, - reverb_freeze_mode, - pitch_shift_semitones, - delay_seconds, - delay_feedback, - delay_mix, - compressor_threshold, - compressor_ratio, - compressor_attack, - compressor_release, - limiter_threshold, - limiter_release_time, - gain_db, - bitcrush_bit_depth, - clipping_threshold, - phaser_rate_hz, - phaser_depth, - phaser_centre_frequency_hz, - phaser_feedback, - phaser_mix -): - if not presets: gr_warning(translations["provide_file_settings"]) - - file = {} - if presets: - with open(os.path.join(configs["presets_path"], presets)) as f: - file = json.load(f) - - gr_info(translations["load_presets"].format(presets=presets)) - - return [ - file.get("exclusive_mode", exclusive_mode), - file.get("vad_enabled", vad_enabled), - file.get("chunk_size", chunk_size), - file.get("pitch", pitch), - file.get("index_strength", index_strength), - file.get("predictor_onnx", predictor_onnx), - file.get("hop_length", hop_length), - file.get("embedder_mode", embedder_mode), - file.get("f0_autotune", f0_autotune), - file.get("proposal_pitch", proposal_pitch), - file.get("f0_autotune_strength", f0_autotune_strength), - file.get("proposal_pitch_threshold", proposal_pitch_threshold), - file.get("rms_mix_rate", rms_mix_rate), - file.get("protect", protect), - file.get("filter_radius", filter_radius), - file.get("silent_threshold", silent_threshold), - file.get("extra_convert_size", extra_convert_size), - file.get("cross_fade_overlap_size", cross_fade_overlap_size), - file.get("vad_sensitivity", vad_sensitivity), - file.get("vad_frame_ms", vad_frame_ms), - file.get("clean_audio", clean_audio), - file.get("clean_strength", clean_strength), - file.get("post_process", post_process), - file.get("embedders_mix", embedders_mix), - file.get("embedders_mix_layers", embedders_mix_layers), - file.get("embedders_mix_ratio", embedders_mix_ratio), - file.get("chorus", chorus), - file.get("distortion", distortion), - file.get("reverb", reverb), - file.get("pitch_shift", pitch_shift), - file.get("delay", delay), - file.get("compressor", compressor), - file.get("limiter", limiter), - file.get("gain", gain), - file.get("bitcrush", bitcrush), - file.get("clipping", clipping), - file.get("phaser", phaser), - file.get("chorus_depth", chorus_depth), - file.get("chorus_rate", chorus_rate), - file.get("chorus_mix", chorus_mix), - file.get("chorus_center_delay", chorus_center_delay), - file.get("chorus_feedback", chorus_feedback), - file.get("distortion_gain", distortion_gain), - file.get("reverb_room_size", reverb_room_size), - file.get("reverb_damping", reverb_damping), - file.get("reverb_wet_gain", reverb_wet_gain), - file.get("reverb_dry_gain", reverb_dry_gain), - file.get("reverb_width", reverb_width), - file.get("reverb_freeze_mode", reverb_freeze_mode), - file.get("pitch_shift_semitones", pitch_shift_semitones), - file.get("delay_seconds", delay_seconds), - file.get("delay_feedback", delay_feedback), - file.get("delay_mix", delay_mix), - file.get("compressor_threshold", compressor_threshold), - file.get("compressor_ratio", compressor_ratio), - file.get("compressor_attack", compressor_attack), - file.get("compressor_release", compressor_release), - file.get("limiter_threshold", limiter_threshold), - file.get("limiter_release_time", limiter_release_time), - file.get("gain_db", gain_db), - file.get("bitcrush_bit_depth", bitcrush_bit_depth), - file.get("clipping_threshold", clipping_threshold), - file.get("phaser_rate_hz", phaser_rate_hz), - file.get("phaser_depth", phaser_depth), - file.get("phaser_centre_frequency_hz", phaser_centre_frequency_hz), - file.get("phaser_feedback", phaser_feedback), - file.get("phaser_mix", phaser_mix) - ] \ No newline at end of file diff --git a/main/app/core/process.py b/main/app/core/process.py deleted file mode 100644 index c8e72955dea44867b1f93110614df079ced3bd24..0000000000000000000000000000000000000000 --- a/main/app/core/process.py +++ /dev/null @@ -1,192 +0,0 @@ -import os -import re -import sys -import shutil -import codecs -import zipfile -import requests - -sys.path.append(os.getcwd()) - -from main.app.variables import logger, translations, configs -from main.app.core.ui import gr_info, gr_warning, gr_error, process_output, replace_punctuation - -def read_docx_text(path): - import xml.etree.ElementTree - - with zipfile.ZipFile(path) as docx: - with docx.open("word/document.xml") as document_xml: - xml_content = document_xml.read() - - WORD_NAMESPACE = codecs.decode( - "{uggc://fpurznf.bcrakzysbezngf.bet/jbeqcebprffvatzy/2006/znva}", - "rot13" - ) - paragraphs = [] - - for paragraph in xml.etree.ElementTree.XML(xml_content).iter(WORD_NAMESPACE + 'p'): - texts = [node.text for node in paragraph.iter(WORD_NAMESPACE + 't') if node.text] - if texts: paragraphs.append(''.join(texts)) - - return '\n'.join(paragraphs) - -def process_input(file_path): - if file_path.endswith(".srt"): - file_contents = "" - elif file_path.endswith(".docx"): - file_contents = read_docx_text(file_path) - else: - try: - with open(file_path, "r", encoding="utf-8") as file: - file_contents = file.read() - except Exception as e: - gr_warning(translations["read_error"]) - - logger.debug(e) - file_contents = "" - - gr_info(translations["upload_success"].format(name=translations["text"])) - return file_contents - -def move_files_from_directory( - src_dir, - dest_weights, - dest_logs, - model_name, - use_orig_weight_name=False -): - for root, _, files in os.walk(src_dir): - for file in files: - file_path = os.path.join(root, file) - - if file.endswith(".index"): - model_log_dir = os.path.join(dest_logs, model_name) - os.makedirs(model_log_dir, exist_ok=True) - - filepath = process_output(os.path.join(model_log_dir, replace_punctuation(file))) - - shutil.move(file_path, filepath) - elif file.endswith(".pth") and not file.startswith("D_") and not file.startswith("G_"): - pth_path = process_output(os.path.join(dest_weights, model_name + ".pth")) - - shutil.move(file_path, pth_path if not use_orig_weight_name else dest_weights) - elif file.endswith(".onnx") and not file.startswith("D_") and not file.startswith("G_"): - pth_path = process_output(os.path.join(dest_weights, model_name + ".onnx")) - - shutil.move(file_path, pth_path if not use_orig_weight_name else dest_weights) - -def extract_name_model(filename): - match = re.search( - r"_([A-Za-z0-9]+)(?=_v\d*)", - replace_punctuation(filename) - ) - - return match.group(1) if match else None - -def save_drop_model(dropboxs): - weight_folder = configs["weights_path"] - logs_folder = configs["logs_path"] - save_model_temp = "save_model_temp" - - if not os.path.exists(weight_folder): os.makedirs(weight_folder, exist_ok=True) - if not os.path.exists(logs_folder): os.makedirs(logs_folder, exist_ok=True) - if not os.path.exists(save_model_temp): os.makedirs(save_model_temp, exist_ok=True) - - try: - for dropbox in dropboxs: - shutil.move(dropbox, save_model_temp) - file_name = os.path.basename(dropbox) - - if file_name.endswith(".zip"): - shutil.unpack_archive( - os.path.join(save_model_temp, file_name), - save_model_temp - ) - - move_files_from_directory( - save_model_temp, - weight_folder, - logs_folder, - file_name.replace(".zip", "") - ) - elif file_name.endswith((".pth", ".onnx")): - output_file = process_output(os.path.join(weight_folder, file_name)) - - shutil.move(os.path.join(save_model_temp, file_name), output_file) - elif file_name.endswith(".index"): - modelname = extract_name_model(file_name) - if modelname is None: - modelname = os.path.splitext(os.path.basename(file_name))[0] - - model_logs = os.path.join(logs_folder, modelname) - if not os.path.exists(model_logs): - os.makedirs(model_logs, exist_ok=True) - - shutil.move( - os.path.join(save_model_temp, file_name), - model_logs - ) - else: - gr_warning(translations["unable_analyze_model"]) - return None - - gr_info(translations["upload_success"].format(name=translations["model"])) - return None - except Exception as e: - gr_error(message=translations["error_occurred"].format(e=e)) - return None - finally: - shutil.rmtree(save_model_temp, ignore_errors=True) - -def zip_file( - modelname, - pth_name, - index_path -): - pth_path = os.path.join(configs["weights_path"], pth_name) - - if ( - not pth_name or - not os.path.exists(pth_path) or - not pth_name.endswith((".pth", ".onnx")) - ): - return gr_warning(translations["provide_file"].format(filename=translations["model"])) - - zip_file_path = os.path.join(configs["logs_path"], modelname, modelname + ".zip") - gr_info(translations["start"].format(start=translations["zip"])) - - with zipfile.ZipFile(zip_file_path, 'w') as zipf: - zipf.write(pth_path, os.path.basename(pth_path)) - if index_path: zipf.write(index_path, os.path.basename(index_path)) - - gr_info(translations["success"]) - - return { - "visible": True, - "value": zip_file_path, - "__type__": "update" - } - -def fetch_pretrained_data(): - try: - response = requests.get( - codecs.decode( - "uggcf://uhttvatsnpr.pb/NauC/Ivrganzrfr-EIP-Cebwrpg/erfbyir/znva/wfba/phfgbz_cergenvarq.wfba", - "rot13" - ) - ) - response.raise_for_status() - - return response.json() - except: - return {} - -def update_sample_rate_dropdown(model): - data = fetch_pretrained_data() - - if model != translations["success"]: - return { - "choices": list(data[model].keys()), - "value": list(data[model].keys())[0], - "__type__": "update" - } \ No newline at end of file diff --git a/main/app/core/realtime.py b/main/app/core/realtime.py deleted file mode 100644 index 56e2dd83655eee445a031e0bf20d4e302b8bfcbc..0000000000000000000000000000000000000000 --- a/main/app/core/realtime.py +++ /dev/null @@ -1,1142 +0,0 @@ -import os -import sys -import time - -sys.path.append(os.getcwd()) - -from main.app.core.ui import gr_info, gr_warning -from main.app.variables import translations, configs, config, logger - -running, callbacks, audio_manager = False, None, None - -PIPELINE_SAMPLE_RATE = 16000 - -interactive_true = {"interactive": True, "__type__": "update"} -interactive_false = {"interactive": False, "__type__": "update"} -callbacks_kwargs = {} - -js_code = """ -window._activeStream = null; -window._audioCtx = null; -window._workletNode = null; -window._playbackNode = null; -window._ws = null; -window.OutputAudioRoute = null; -window.MonitorAudioRoute = null; -window.lastSend = 0; -window.responseMs = 0; - -function setStatus(msg, use_alert = true) { - const realtimeStatus = document.querySelector("#realtime-status-info h2.output-class"); - if (use_alert) alert(msg); - - if (realtimeStatus) { - realtimeStatus.innerText = msg; - realtimeStatus.style.whiteSpace = "nowrap"; - realtimeStatus.style.textAlign = "center"; - } -} - -async function addModuleFromString(ctx, codeStr) { - const blob = new Blob([codeStr], {type: 'application/javascript'}); - const url = URL.createObjectURL(blob); - - await ctx.audioWorklet.addModule(url); - URL.revokeObjectURL(url); -}; - -function createOutputRoute(audioCtx, playbackNode, sinkId, gainValue = 1.0) { - const dest = audioCtx.createMediaStreamDestination(); - const gainNode = audioCtx.createGain(); - gainNode.gain.value = gainValue; - - playbackNode.connect(gainNode); - gainNode.connect(dest); - - const el = document.createElement('audio'); - el.autoplay = true; - el.srcObject = dest.stream; - el.style.display = 'none'; - document.body.appendChild(el); - - if (el.setSinkId) el.setSinkId(sinkId).catch(err => console.error(err)); - return { dest, gainNode, el }; -} - -const inputWorkletSource = ` - class InputProcessor extends AudioWorkletProcessor { - constructor() { - super(); - this.buffer = new Float32Array(0); - this.block_frame = 128; - this.port.onmessage = (e) => { - if (e.data && e.data.block_frame) this.block_frame = e.data.block_frame; - }; - } - - process(inputs) { - const input = inputs[0]; - if (!input || !input[0]) return true; - const frame = input[0]; - - const newBuf = new Float32Array(this.buffer.length + frame.length); - newBuf.set(this.buffer, 0); - newBuf.set(frame, this.buffer.length); - this.buffer = newBuf; - - while (this.buffer.length >= this.block_frame) { - const chunk = this.buffer.slice(0, this.block_frame); - - this.port.postMessage({chunk}, [chunk.buffer]); - this.buffer = this.buffer.slice(this.block_frame); - } - - return true; - } - } - registerProcessor('input-processor', InputProcessor); - `; - - const playbackWorkletSource = ` - class PlaybackProcessor extends AudioWorkletProcessor { - constructor(options) { - super(options); - const bufferSize = options.processorOptions && options.processorOptions.bufferSize ? options.processorOptions.bufferSize: 98304; - this.buffer = new Float32Array(bufferSize); - this.bufferCapacity = bufferSize; - this.writePointer = 0; - this.readPointer = 0; - this.availableSamples = 0; - this.port.onmessage = (e) => { - if (e.data && e.data.chunk) { - const chunk = new Float32Array(e.data.chunk); - const chunkSize = chunk.length; - - if (this.availableSamples + chunkSize > this.bufferCapacity) return; - - for (let i = 0; i < chunkSize; i++) { - this.buffer[this.writePointer] = chunk[i]; - this.writePointer = (this.writePointer + 1) % this.bufferCapacity; - } - - this.availableSamples += chunkSize; - } - }; - } - - process(inputs, outputs) { - const output = outputs[0]; - if (!output || !output[0]) return true; - - const frame = output[0]; - const frameSize = frame.length; - - if (this.availableSamples >= frameSize) { - for (let i = 0; i < frameSize; i++) { - frame[i] = this.buffer[this.readPointer]; - this.readPointer = (this.readPointer + 1) % this.bufferCapacity; - } - this.availableSamples -= frameSize; - } else { - frame.fill(0); - } - - if (output.length > 1) output[1].set(output[0]); - return true; - } - } - registerProcessor('playback-processor', PlaybackProcessor); - `; - -window.getAudioDevices = async function() { - if (!navigator.mediaDevices) { - setStatus("__MEDIA_DEVICES__"); - return {"inputs": {}, "outputs": {}}; - } - - try { - await navigator.mediaDevices.getUserMedia({ audio: true }); - } catch (err) { - console.error(err); - setStatus("__MIC_INACCESSIBLE__") - - return {"inputs": {}, "outputs": {}}; - } - - const devices = await navigator.mediaDevices.enumerateDevices(); - const inputs = {}; - const outputs = {}; - - for (const device of devices) { - if (device.kind === "audioinput") { - inputs[device.label] = device.deviceId - } else if (device.kind === "audiooutput") { - outputs[device.label] = device.deviceId - } - } - - if (!Object.keys(inputs).length && !Object.keys(outputs).length) return {"inputs": {}, "outputs": {}}; - return {"inputs": inputs, "outputs": outputs}; -}; - -window.StreamAudioRealtime = async function( - monitor, - vad_enabled, - input_audio_device, - output_audio_device, - monitor_output_device, - input_audio_gain, - output_audio_gain, - monitor_audio_gain, - chunk_size, - pitch, - model_pth, - model_index, - index_strength, - predictor_onnx, - f0_method, - hop_length, - embed_mode, - embedders, - custom_embedders, - f0_autotune, - proposal_pitch, - f0_autotune_strength, - proposal_pitch_threshold, - rms_mix_rate, - protect, - filter_radius, - silent_threshold, - extra_convert_size, - cross_fade_overlap_size, - vad_sensitivity, - vad_frame_ms, - clean_audio, - clean_strength, - exclusive_mode, - post_process, - sid, - noise_scale, - embedders_mix, - embedders_mix_layers, - embedders_mix_ratio, - chorus, - distortion, - reverb, - pitch_shift, - delay, - compressor, - limiter, - gain, - bitcrush, - clipping, - phaser, - chorus_depth, - chorus_rate, - chorus_mix, - chorus_center_delay, - chorus_feedback, - distortion_gain, - reverb_room_size, - reverb_damping, - reverb_wet_gain, - reverb_dry_gain, - reverb_width, - reverb_freeze_mode, - pitch_shift_semitones, - delay_seconds, - delay_feedback, - delay_mix, - compressor_threshold, - compressor_ratio, - compressor_attack, - compressor_release, - limiter_threshold, - limiter_release_time, - gain_db, - bitcrush_bit_depth, - clipping_threshold, - phaser_rate_hz, - phaser_depth, - phaser_centre_frequency_hz, - phaser_feedback, - phaser_mix, - nprobe -) { - const SampleRate = 48000; - const ReadChunkSize = Math.round(chunk_size * SampleRate / 1000 / 128); - const block_frame = parseInt(ReadChunkSize) * 128; - const ButtonState = { start_button: true, stop_button: false }; - const devices = await window.getAudioDevices(); - - input_audio_device = devices["inputs"][input_audio_device]; - output_audio_device = devices["outputs"][output_audio_device]; - if (monitor && devices["outputs"][monitor_output_device]) monitor_output_device = devices["outputs"][monitor_output_device]; - - try { - if (!input_audio_device || !output_audio_device) { - setStatus("__PROVIDE_AUDIO_DEVICE__"); - return ButtonState; - } - - if (monitor && !monitor_output_device) { - setStatus("__PROVIDE_MONITOR_DEVICE__"); - return ButtonState; - } - - if (!model_pth) { - setStatus("__PROVIDE_MODEL__") - return ButtonState; - } - - setStatus("__START_REALTIME__", false) - - const stream = await navigator.mediaDevices.getUserMedia({ - audio: { - deviceId: { exact: input_audio_device }, - channelCount: 1, - sampleRate: SampleRate, - echoCancellation: !exclusive_mode, - noiseSuppression: !exclusive_mode, - autoGainControl: !exclusive_mode - } - }); - - let latencyHint = "playback"; - if (exclusive_mode) latencyHint = "interactive"; - - window._activeStream = stream; - window._audioCtx = new AudioContext({ sampleRate: SampleRate, latencyHint: latencyHint }); - - await addModuleFromString(window._audioCtx, inputWorkletSource); - await addModuleFromString(window._audioCtx, playbackWorkletSource); - - const src = window._audioCtx.createMediaStreamSource(stream); - const inputNode = new AudioWorkletNode(window._audioCtx, 'input-processor'); - const playbackNode = new AudioWorkletNode(window._audioCtx, 'playback-processor', { - processorOptions: { - bufferSize: block_frame * 2 - } - }); - - inputNode.port.postMessage({ block_frame: block_frame }); - src.connect(inputNode); - - window.OutputAudioRoute = createOutputRoute(window._audioCtx, playbackNode, output_audio_device, output_audio_gain / 100); - if (monitor && monitor_output_device) window.MonitorAudioRoute = createOutputRoute(window._audioCtx, playbackNode, monitor_output_device, monitor_audio_gain / 100); - - const protocol = (location.protocol === "https:") ? "wss:" : "ws:"; - const wsUrl = protocol + '//' + location.hostname + `:${location.port}` + '/api/ws-audio'; - const ws = new WebSocket(wsUrl); - - ButtonState.start_button = false; - ButtonState.stop_button = true; - - ws.binaryType = "arraybuffer"; - window._ws = ws; - - ws.onopen = () => { - console.log("__WS_CONNECTED__") - - ws.send( - JSON.stringify({ - type: 'init', - chunk_size: ReadChunkSize, - embedders: embedders, - model_pth: model_pth, - custom_embedders: custom_embedders, - cross_fade_overlap_size: cross_fade_overlap_size, - extra_convert_size: extra_convert_size, - model_index: model_index, - f0_method: f0_method, - predictor_onnx: predictor_onnx, - embedders_mode: embed_mode, - hop_length: hop_length, - silent_threshold: silent_threshold, - vad_enabled: vad_enabled, - vad_sensitivity: vad_sensitivity, - vad_frame_ms: vad_frame_ms, - clean_audio: clean_audio, - clean_strength: clean_strength, - f0_up_key: pitch, - index_rate: index_strength, - protect: protect, - filter_radius: filter_radius, - rms_mix_rate: rms_mix_rate, - f0_autotune: f0_autotune, - f0_autotune_strength: f0_autotune_strength, - proposal_pitch: proposal_pitch, - proposal_pitch_threshold: proposal_pitch_threshold, - input_audio_gain: input_audio_gain, - post_process: post_process, - sid: sid, - noise_scale: noise_scale, - embedders_mix: embedders_mix, - embedders_mix_layers: embedders_mix_layers, - embedders_mix_ratio: embedders_mix_ratio, - nprobe: nprobe, - kwargs: { - chorus: chorus, - distortion: distortion, - reverb: reverb, - pitch_shift: pitch_shift, - delay: delay, - compressor: compressor, - limiter: limiter, - gain: gain, - bitcrush: bitcrush, - clipping: clipping, - phaser: phaser, - chorus_depth: chorus_depth, - chorus_rate: chorus_rate, - chorus_mix: chorus_mix, - chorus_delay: chorus_center_delay, - chorus_feedback: chorus_feedback, - distortion_gain: distortion_gain, - reverb_room_size: reverb_room_size, - reverb_damping: reverb_damping, - reverb_wet_level: reverb_wet_gain, - reverb_dry_level: reverb_dry_gain, - reverb_width: reverb_width, - reverb_freeze_mode: reverb_freeze_mode, - pitch_shift_semitones: pitch_shift_semitones, - delay_seconds: delay_seconds, - delay_feedback: delay_feedback, - delay_mix: delay_mix, - compressor_threshold: compressor_threshold, - compressor_ratio: compressor_ratio, - compressor_attack: compressor_attack, - compressor_release: compressor_release, - limiter_threshold: limiter_threshold, - limiter_release: limiter_release_time, - gain_db: gain_db, - bitcrush_bit_depth: bitcrush_bit_depth, - clipping_threshold: clipping_threshold, - phaser_rate_hz: phaser_rate_hz, - phaser_depth: phaser_depth, - phaser_centre_frequency_hz: phaser_centre_frequency_hz, - phaser_feedback: phaser_feedback, - phaser_mix: phaser_mix - } - }) - ); - }; - - inputNode.port.onmessage = (e) => { - const chunk = e.data && e.data.chunk; - - if (!chunk) return; - if (ws.readyState === WebSocket.OPEN) { - window.lastSend = performance.now(); - ws.send(chunk); - } - }; - - ws.onmessage = (ev) => { - if (typeof ev.data === 'string') { - const msg = JSON.parse(ev.data); - - if (msg.type === 'latency') setStatus(`__LATENCY__: ${msg.value.toFixed(2)} ms | __VOLUME__: ${msg.volume.toFixed(2)} dB | __RESPONSE__: ${window.responseMs.toFixed(2)} ms`, use_alert=false) - if (msg.type === 'warnings') { - setStatus(msg.value); - StopAudioStream(); - } - - return; - } - - const ab = ev.data; - playbackNode.port.postMessage({ chunk: ab }, [ab]); - window.responseMs = performance.now() - window.lastSend; - }; - - ws.onclose = () => console.log("__WS_CLOSED__"); - window._workletNode = inputNode; - window._playbackNode = playbackNode; - - if (window._audioCtx.state === 'suspended') await window._audioCtx.resume(); - - console.log("__REALTIME_STARTED__"); - return ButtonState; - } catch (err) { - console.error("__ERROR__", err); - alert("__ERROR__" + err.message); - - return StopAudioStream(); - } -}; - -window.ChangeConfig = async function(value, key, if_kwargs=false) { - if (key === "output_audio_gain") { - window.OutputAudioRoute.gainNode.gain.value = value / 100 - } else if (key == "monitor_audio_gain") { - if (window.MonitorAudioRoute) window.MonitorAudioRoute.gainNode.gain.value = value / 100 - } else { - const protocol = (location.protocol === "https:") ? "wss:" : "ws:"; - const wsUrl = protocol + '//' + location.hostname + `:${location.port}` + '/api/change-config'; - const ws = new WebSocket(wsUrl); - - ws.binaryType = "arraybuffer"; - ws.onopen = () => { - ws.send( - JSON.stringify({ - type: 'init', - key: key, - value: value, - if_kwargs: if_kwargs - }) - ); - - ws.close(); - }; - } -} - -window.StopAudioStream = async function() { - try { - if (window._ws) { - window._ws.close(); - window._ws = null; - } - - if (window._activeStream) { - window._activeStream.getTracks().forEach(t => t.stop()); - window._activeStream = null; - } - - if (window._workletNode) { - window._workletNode.disconnect(); - window._workletNode = null; - } - - if (window._playbackNode) { - window._playbackNode.disconnect(); - window._playbackNode = null; - } - - if (window._audioCtx) { - await window._audioCtx.close(); - window._audioCtx = null; - } - - if (window.OutputAudioRoute) window.OutputAudioRoute = null; - if (window.MonitorAudioRoute) window.MonitorAudioRoute = null; - - document.querySelectorAll('audio').forEach(a => a.remove()); - setStatus("__REALTIME_HAS_STOP__", false); - - return {"start_button": true, "stop_button": false}; - } catch (e) { - setStatus(`__ERROR__ ${e}`); - - return {"start_button": false, "stop_button": true} - } -}; - -window.SoundfileRecordAudio = async function (RecordButton, RecordAudioPath, ExportFormat) { - const protocol = (location.protocol === "https:") ? "https:" : "http:"; - const url = protocol + '//' + location.hostname + `:${location.port}` + '/api/record'; - - const res = await fetch(url, { - method: "POST", - headers: { - "Content-Type": "application/json" - }, - body: JSON.stringify({ - record_button: RecordButton, - record_audio_path: RecordAudioPath, - export_format: ExportFormat - }) - }); - - const msg = await res.json(); - - if (msg.type === "info" || msg.type === "warnings") { - alert(msg.value); - - return { - "button": msg.button, - "path": msg.path - }; - } -}; -""".replace( - "__MEDIA_DEVICES__", translations["media_devices"] -).replace( - "__MIC_INACCESSIBLE__", translations["mic_inaccessible"] -).replace( - "__PROVIDE_AUDIO_DEVICE__", translations["provide_audio_device"] -).replace( - "__PROVIDE_MONITOR_DEVICE__", translations["provide_monitor_device"] -).replace( - "__START_REALTIME__", translations["start_realtime"] -).replace( - "__LATENCY__", translations['latency'] -).replace( - "__WS_CONNECTED__", translations["ws_connected"] -).replace( - "__WS_CLOSED__", translations["ws_closed"] -).replace( - "__REALTIME_STARTED__", translations["realtime_is_ready"] -).replace( - "__ERROR__", translations["error_occurred"].format(e="") -).replace( - "__REALTIME_HAS_STOP__", translations["realtime_has_stop"] -).replace( - "__PROVIDE_MODEL__", translations["provide_file"].format(filename=translations["model"]) -).replace( - "__VOLUME__", translations["volume"] -).replace( - "__RESPONSE__", translations["response"] -) - -def realtime_start( - monitor, - exclusive_mode, - vad_enabled, - input_audio_device, - output_audio_device, - monitor_output_device, - input_audio_gain, - output_audio_gain, - monitor_audio_gain, - input_asio_channels, - output_asio_channels, - monitor_asio_channels, - input_audio_sample_rate, - output_audio_sample_rate, - monitor_audio_sample_rate, - chunk_size, - pitch, - model_pth, - model_index, - index_strength, - predictor_onnx, - f0_method, - hop_length, - embed_mode, - embedders, - custom_embedders, - f0_autotune, - proposal_pitch, - f0_autotune_strength, - proposal_pitch_threshold, - rms_mix_rate, - protect, - filter_radius, - silent_threshold, - extra_convert_size, - cross_fade_overlap_size, - vad_sensitivity, - vad_frame_ms, - clean_audio, - clean_strength, - post_process, - sid, - noise_scale, - embedders_mix, - embedders_mix_layers, - embedders_mix_ratio, - chorus, - distortion, - reverb, - pitch_shift, - delay, - compressor, - limiter, - gain, - bitcrush, - clipping, - phaser, - chorus_depth, - chorus_rate, - chorus_mix, - chorus_center_delay, - chorus_feedback, - distortion_gain, - reverb_room_size, - reverb_damping, - reverb_wet_gain, - reverb_dry_gain, - reverb_width, - reverb_freeze_mode, - pitch_shift_semitones, - delay_seconds, - delay_feedback, - delay_mix, - compressor_threshold, - compressor_ratio, - compressor_attack, - compressor_release, - limiter_threshold, - limiter_release_time, - gain_db, - bitcrush_bit_depth, - clipping_threshold, - phaser_rate_hz, - phaser_depth, - phaser_centre_frequency_hz, - phaser_feedback, - phaser_mix, - nprobe = 1 -): - global running, callbacks, audio_manager, callbacks_kwargs - running = True - - gr_info(translations["start_realtime"]) - - yield ( - translations["start_realtime"], - interactive_false, - interactive_true - ) - - if not input_audio_device or not output_audio_device: - gr_warning(translations["provide_audio_device"]) - - yield ( - translations["provide_audio_device"], - interactive_true, - interactive_false - ) - return - - if monitor and not monitor_output_device: - gr_warning(translations["provide_monitor_device"]) - - yield ( - translations["provide_monitor_device"], - interactive_true, - interactive_false - ) - return - - model_pth = os.path.join(configs["weights_path"], model_pth) if not os.path.exists(model_pth) else model_pth - embedder_model = (embedders if embedders != "custom" else custom_embedders) - - if ( - not model_pth or - not os.path.exists(model_pth) or - os.path.isdir(model_pth) or - not model_pth.endswith((".pth", ".onnx")) - ): - gr_warning(translations["provide_file"].format(filename=translations["model"])) - - yield ( - translations["provide_file"].format(filename=translations["model"]), - interactive_true, - interactive_false - ) - return - - input_is_asio = "ASIO" in input_audio_device - output_is_asio = "ASIO" in output_audio_device - monitor_is_asio = "ASIO" in monitor_output_device - - if input_is_asio or output_is_asio or monitor_is_asio: - import main.library.audio.sounddevice as sd - - sd.terminate() - sd.initialize() - - import main.app.core.ui as ui - - input_device_id = ui.input_channels_map[input_audio_device][0] - output_device_id = ui.output_channels_map[output_audio_device][0] - output_monitor_id = ui.output_channels_map[monitor_output_device][0] if monitor else None - - input_audio_gain /= 100.0 - output_audio_gain /= 100.0 - monitor_audio_gain /= 100.0 - - chunk_size = int(chunk_size * input_audio_sample_rate / 1000 / 128) - - from main.inference.realtime.callbacks import AudioCallbacks - - callbacks_kwargs = { - "pass_through": False, - "read_chunk_size": chunk_size, - "cross_fade_overlap_size": cross_fade_overlap_size, - "input_sample_rate": input_audio_sample_rate, - "output_sample_rate": input_audio_sample_rate, - "extra_convert_size": extra_convert_size, - "model_path": model_pth, - "index_path": model_index, - "f0_method": f0_method, - "predictor_onnx": predictor_onnx, - "embedder_model": embedder_model, - "embedders_mode": embed_mode, - "sample_rate": PIPELINE_SAMPLE_RATE, - "hop_length": hop_length, - "silent_threshold": silent_threshold, - "f0_up_key": pitch, - "index_rate": index_strength, - "protect": protect, - "filter_radius": filter_radius, - "rms_mix_rate": rms_mix_rate, - "f0_autotune": f0_autotune, - "f0_autotune_strength": f0_autotune_strength, - "proposal_pitch": proposal_pitch, - "proposal_pitch_threshold": proposal_pitch_threshold, - "input_audio_gain": input_audio_gain, - "output_audio_gain": output_audio_gain, - "monitor_audio_gain": monitor_audio_gain, - "monitor": monitor, - "vad_enabled": vad_enabled, - "vad_sensitivity": vad_sensitivity, - "vad_frame_ms": vad_frame_ms, - "clean_audio": clean_audio, - "clean_strength": clean_strength, - "post_process": post_process, - "sid": sid, - "noise_scale": noise_scale, - "nprobe": nprobe, - "embedders_mix": embedders_mix, - "embedders_mix_layers": embedders_mix_layers, - "embedders_mix_ratio": embedders_mix_ratio, - "kwargs": { - "chorus": chorus, - "distortion": distortion, - "reverb": reverb, - "pitch_shift": pitch_shift, - "delay": delay, - "compressor": compressor, - "limiter": limiter, - "gain": gain, - "bitcrush": bitcrush, - "clipping": clipping, - "phaser": phaser, - "chorus_depth": chorus_depth, - "chorus_rate": chorus_rate, - "chorus_mix": chorus_mix, - "chorus_delay": chorus_center_delay, - "chorus_feedback": chorus_feedback, - "distortion_gain": distortion_gain, - "reverb_room_size": reverb_room_size, - "reverb_damping": reverb_damping, - "reverb_wet_level": reverb_wet_gain, - "reverb_dry_level": reverb_dry_gain, - "reverb_width": reverb_width, - "reverb_freeze_mode": reverb_freeze_mode, - "pitch_shift_semitones": pitch_shift_semitones, - "delay_seconds": delay_seconds, - "delay_feedback": delay_feedback, - "delay_mix": delay_mix, - "compressor_threshold": compressor_threshold, - "compressor_ratio": compressor_ratio, - "compressor_attack": compressor_attack, - "compressor_release": compressor_release, - "limiter_threshold": limiter_threshold, - "limiter_release": limiter_release_time, - "gain_db": gain_db, - "bitcrush_bit_depth": bitcrush_bit_depth, - "clipping_threshold": clipping_threshold, - "phaser_rate_hz": phaser_rate_hz, - "phaser_depth": phaser_depth, - "phaser_centre_frequency_hz": phaser_centre_frequency_hz, - "phaser_feedback": phaser_feedback, - "phaser_mix": phaser_mix - } - } - - callbacks = AudioCallbacks(**callbacks_kwargs) - - def stop_with_error(): - global running - - realtime_stop() - running = False - - return ( - translations["realtime_has_stop_with_error"], - interactive_true, - interactive_false - ) - - try: - audio_manager = callbacks.audio - audio_manager.start( - input_device_id=input_device_id, - output_device_id=output_device_id, - output_monitor_id=output_monitor_id, - exclusive_mode=exclusive_mode, - asio_input_channel=input_asio_channels, - asio_output_channel=output_asio_channels, - asio_output_monitor_channel=monitor_asio_channels, - read_chunk_size=chunk_size, - input_audio_sample_rate=input_audio_sample_rate, - output_audio_sample_rate=output_audio_sample_rate, - output_monitor_sample_rate=monitor_audio_sample_rate - ) - except Exception as e: - import traceback - - logger.error(e) - logger.debug(traceback.format_exc()) - - return stop_with_error() - - gr_info(translations["realtime_is_ready"]) - - while running and callbacks is not None and audio_manager is not None: - time.sleep(0.1) - if hasattr(callbacks, "audio") and hasattr(callbacks.audio, "performance") and hasattr(callbacks.audio, "volume"): - yield ( - f"{translations['latency']}: {callbacks.audio.performance:.2f} ms | {translations['volume']}: {callbacks.audio.volume:.2f} dB", - interactive_false, - interactive_true - ) - - return ( - translations["realtime_has_stop"], - interactive_true, - interactive_false - ) - -def change_callbacks_config(): - global callbacks - - if running and audio_manager is not None and callbacks is not None: - crossfade_frame = int(callbacks_kwargs.get("cross_fade_overlap_size", 0.1) * callbacks.input_sample_rate) - extra_frame = int(callbacks_kwargs.get("extra_convert_size", 0.5) * callbacks.input_sample_rate) - - if ( - callbacks.vc.crossfade_frame != crossfade_frame or - callbacks.vc.extra_frame != extra_frame - ): - del ( - callbacks.vc.fade_in_window, - callbacks.vc.fade_out_window, - callbacks.vc.sola_buffer - ) - - callbacks.vc.vc_model.realloc( - callbacks.vc.block_frame, - callbacks.vc.extra_frame, - callbacks.vc.crossfade_frame, - callbacks.vc.sola_search_frame, - ) - callbacks.vc.generate_strength() - - callbacks.vc.vc_model.input_sensitivity = 10 ** (callbacks_kwargs.get("silent_threshold", -90) / 20) - - vad_enabled = callbacks_kwargs.get("vad_enabled", True) - sensitivity_mode = callbacks_kwargs.get("vad_sensitivity", 3) - vad_frame_ms = callbacks_kwargs.get("vad_frame_ms", 30) - - if vad_enabled is False: - callbacks.vc.vc_model.vad = None - elif vad_enabled and callbacks.vc.vc_model.vad is None: - from main.inference.realtime.vad_utils import VADProcessor - - callbacks.vc.vc_model.vad = VADProcessor( - sensitivity_mode=sensitivity_mode, - sample_rate=callbacks.vc.vc_model.sample_rate, - frame_duration_ms=vad_frame_ms - ) - - if callbacks.vc.vc_model.vad is not None: - callbacks.vc.vc_model.vad.vad.set_mode(sensitivity_mode) - callbacks.vc.vc_model.vad.frame_length = int(callbacks.vc.vc_model.sample_rate * (vad_frame_ms / 1000.0)) - - clean_audio = callbacks_kwargs.get("clean_audio", False) - clean_strength = callbacks_kwargs.get("clean_strength", 0.5) - - if clean_audio is False: - callbacks.vc.vc_model.tg = None - elif clean_audio and callbacks.vc.vc_model.tg is None: - from main.library.audio.noisereduce import TorchGate - - callbacks.vc.vc_model.tg = ( - TorchGate( - callbacks.vc.vc_model.pipeline.tgt_sr, - prop_decrease=clean_strength, - ).to(config.device) - ) - - if callbacks.vc.vc_model.tg is not None: - callbacks.vc.vc_model.tg.prop_decrease = clean_strength - - post_process = callbacks_kwargs.get("post_process", False) - kwargs = callbacks_kwargs.get("kwargs", {}) - - if post_process is False: - callbacks.vc.vc_model.board = None - callbacks.vc.vc_model.kwargs = None - elif post_process and callbacks.vc.vc_model.kwargs != kwargs: - new_board = callbacks.vc.vc_model.setup_pedalboard(**kwargs) - callbacks.vc.vc_model.board = new_board - callbacks.vc.vc_model.kwargs = kwargs.copy() - - callbacks.audio.f0_up_key = callbacks_kwargs.get("f0_up_key", 0) - callbacks.audio.index_rate = callbacks_kwargs.get("index_rate", 0.75) - callbacks.audio.protect = callbacks_kwargs.get("protect", 0.5) - callbacks.audio.rms_mix_rate = callbacks_kwargs.get("rms_mix_rate", 1) - - callbacks.audio.f0_autotune = callbacks_kwargs.get("f0_autotune", False) - callbacks.audio.f0_autotune_strength = callbacks_kwargs.get("f0_autotune_strength", 1.0) - callbacks.audio.proposal_pitch = callbacks_kwargs.get("proposal_pitch", False) - callbacks.audio.proposal_pitch_threshold = callbacks_kwargs.get("proposal_pitch_threshold", 155.0) - - callbacks.audio.input_audio_gain = callbacks_kwargs.get("input_audio_gain", 1.0) - callbacks.audio.output_audio_gain = callbacks_kwargs.get("output_audio_gain", 1.0) - callbacks.audio.monitor_audio_gain = callbacks_kwargs.get("monitor_audio_gain", 1.0) - - callbacks.audio.embedders_mix = callbacks_kwargs.get("embedders_mix", False) - callbacks.audio.embedders_mix_layers = callbacks_kwargs.get("embedders_mix_layers", 9) - callbacks.audio.embedders_mix_ratio = callbacks_kwargs.get("embedders_mix_ratio", 0.5) - - noise_scale = callbacks_kwargs.get("noise_scale", 0.35) - model_pth = callbacks_kwargs.get("model_path", callbacks.vc.vc_model.model_path) - model_pth = os.path.join(configs["weights_path"], model_pth) if not os.path.exists(model_pth) else model_pth - - if model_pth and callbacks.vc.vc_model.model_path != model_pth: - callbacks.vc.vc_model.model_path = model_pth - callbacks.vc.vc_model.pipeline.vc.setup(model_pth, noise_scale) - callbacks.vc.vc_model.pipeline.version = callbacks.vc.vc_model.pipeline.vc.version - callbacks.vc.vc_model.pipeline.energy = callbacks.vc.vc_model.pipeline.vc.energy - callbacks.vc.vc_model.pipeline.rms = callbacks.vc.vc_model.pipeline.setup_rms() if callbacks.vc.vc_model.pipeline.vc.energy else None - - sid = callbacks_kwargs.get("sid", callbacks.vc.vc_model.pipeline.sid) - if callbacks.vc.vc_model.pipeline.sid != sid: - import torch - callbacks.vc.vc_model.pipeline.torch_sid = torch.tensor( - [sid], device=callbacks.vc.vc_model.pipeline.device, dtype=torch.int64 - ) - - index_path = callbacks_kwargs.get("index_path", None) - if index_path: - if callbacks.vc.vc_model.index_path != index_path: - from main.library.utils import load_faiss_index - - nprobe = callbacks_kwargs.get("nprobe", 1) - index, index_reconstruct = load_faiss_index( - index_path.strip().strip('"').strip("\n").strip('"').strip().replace("trained", "added"), - nprobe=nprobe - ) - - callbacks.vc.vc_model.pipeline.index = index - if callbacks.vc.vc_model.pipeline.index.index is not None: callbacks.vc.vc_model.pipeline.index.index.nprobe = nprobe - callbacks.vc.vc_model.pipeline.big_tsr = index_reconstruct - callbacks.vc.vc_model.index_path = index_path - else: - callbacks.vc.vc_model.pipeline.index = None - callbacks.vc.vc_model.pipeline.big_tsr = None - callbacks.vc.vc_model.index_path = None - - f0_method = callbacks_kwargs.get("f0_method", callbacks.vc.vc_model.pipeline.f0_method) - predictor_onnx = callbacks_kwargs.get("predictor_onnx", callbacks.vc.vc_model.pipeline.predictor.predictor_onnx) - embedders = callbacks_kwargs.get("embedder_model", callbacks.vc.vc_model.embedder_model) - embedders_mode = callbacks_kwargs.get("embedders_mode", callbacks.vc.vc_model.embedders_mode) - custom_embedders = callbacks_kwargs.get("embedder_model_custom", None) - embedder_model = (embedders if embedders != "custom" else custom_embedders) - - from main.library.utils import check_assets - check_assets(f0_method, embedders, predictor_onnx, embedders_mode) - - if ( - callbacks.vc.vc_model.pipeline.f0_method != f0_method or - callbacks.vc.vc_model.pipeline.predictor.predictor_onnx != predictor_onnx - ): - old_predictor = callbacks.vc.vc_model.pipeline.predictor - del old_predictor - - callbacks.vc.vc_model.pipeline.predictor = callbacks.vc.vc_model.pipeline.setup_predictor(PIPELINE_SAMPLE_RATE, callbacks_kwargs.get("hop_length", callbacks.vc.vc_model.pipeline.predictor.hop_length), predictor_onnx) - callbacks.vc.vc_model.pipeline.f0_method = f0_method - callbacks.vc.vc_model.pipeline.predictor.predictor_onnx = predictor_onnx - - if embedder_model: - if ( - callbacks.vc.vc_model.embedder_model != embedder_model or - callbacks.vc.vc_model.embedders_mode != embedders_mode - ): - old_embedder = callbacks.vc.vc_model.pipeline.embedder - del old_embedder - - callbacks.vc.vc_model.pipeline.embedder = callbacks.vc.vc_model.pipeline.setup_embedder(embedder_model, embedders_mode) - callbacks.vc.vc_model.embedder_model = embedder_model - callbacks.vc.vc_model.embedders_mode = embedders_mode - - if ( - callbacks.vc.vc_model.pipeline.vc.architecture == "SVC" and - callbacks.vc.vc_model.pipeline.vc.net_g.noise_scale != noise_scale - ): - callbacks.vc.vc_model.pipeline.vc.net_g.noise_scale = noise_scale - -def change_config(value, key, if_kwargs=False): - global callbacks_kwargs - - if running and audio_manager is not None and callbacks is not None: - if if_kwargs: - callbacks_kwargs["kwargs"][key] = value - else: - callbacks_kwargs[key] = value - - change_callbacks_config() - -def realtime_stop(): - global running, callbacks, audio_manager - - if running and audio_manager is not None and callbacks is not None: - gr_info(translations["stop_realtime"]) - - audio_manager.stop() - running = False - - if hasattr(callbacks.audio, "performance"): del callbacks.audio.performance - if hasattr(callbacks.audio, "volume"): del callbacks.audio.volume - - del callbacks.vc.vc_model.pipeline, callbacks.vc.vc_model - del audio_manager, callbacks - - audio_manager = callbacks = None - gr_info(translations["realtime_has_stop"]) - - from main.library.utils import clear_gpu_cache - clear_gpu_cache() - - return ( - translations["realtime_has_stop"], - interactive_true, - interactive_false - ) - else: - gr_warning(translations["realtime_not_found"]) - - return ( - translations["realtime_not_found"], - interactive_true, - interactive_false - ) - -def soundfile_record_audio( - record_button, - record_audio_path = None, - export_format = "wav" -): - global callbacks - - if running and audio_manager is not None and callbacks is not None: - if record_button == translations["start_record"]: - gr_info(translations["starting_record"]) - - if not record_audio_path: - record_audio_path = os.path.join(configs["audios_path"], "record_audio.wav") - - callbacks.vc.record_audio = True - callbacks.vc.record_audio_path = record_audio_path - callbacks.vc.export_format = export_format - callbacks.vc.setup_soundfile_record() - - return translations["stop_record"], None - else: - gr_info(translations["stopping_record"]) - - callbacks.vc.record_audio = False - callbacks.vc.record_audio_path = None - callbacks.vc.soundfile = None - - return translations["start_record"], record_audio_path - - gr_warning(translations["realtime_not_found"]) - return translations["start_record"], None \ No newline at end of file diff --git a/main/app/core/restart.py b/main/app/core/restart.py deleted file mode 100644 index 157fd096b3afa3b3bb4903808c04fac5ca9ee860..0000000000000000000000000000000000000000 --- a/main/app/core/restart.py +++ /dev/null @@ -1,43 +0,0 @@ -import os -import sys -import json -import platform -import subprocess - -sys.path.append(os.getcwd()) - -from main.app.core.ui import gr_info -from main.app.variables import python, translations, configs_json - -def restart_app(app): - gr_info(translations["restart"]) - os.system("cls" if platform.system() == "Windows" else "clear") - - app.close() - subprocess.run([ - python, - os.path.join("main", "app", "app.py") - ] + [arg for arg in sys.argv[1:] if arg != "--open"]) - -def restart(lang, theme, font, app): - configs = json.load(open(configs_json, "r")) - - if lang != configs["language"]: - configs["language"] = lang - - with open(configs_json, "w") as f: - json.dump(configs, f, indent=4) - - if theme != configs["theme"]: - configs["theme"] = theme - - with open(configs_json, "w") as f: - json.dump(configs, f, indent=4) - - if font != configs["font"]: - configs["font"] = font - - with open(configs_json, "w") as f: - json.dump(configs, f, indent=4) - - restart_app(app) \ No newline at end of file diff --git a/main/app/core/separate.py b/main/app/core/separate.py deleted file mode 100644 index b4c2e0fd4ac5a0bd2f1692e0e74027e4a5f7ee3f..0000000000000000000000000000000000000000 --- a/main/app/core/separate.py +++ /dev/null @@ -1,96 +0,0 @@ -import os -import sys -import subprocess - -sys.path.append(os.getcwd()) - -from main.app.core.ui import gr_info, gr_warning -from main.app.variables import python, translations, configs - -def separate_music( - input_path, - output_dirs, - export_format, - model_name, - karaoke_model, - reverb_model, - denoise_model, - sample_rate, - shifts, - batch_size, - overlap, - aggression, - hop_length, - window_size, - segments_size, - post_process_threshold, - enable_tta, - enable_denoise, - high_end_process, - enable_post_process, - separate_backing, - separate_reverb -): - output_dirs = os.path.dirname(output_dirs) or output_dirs - - if not input_path or not os.path.exists(input_path) or os.path.isdir(input_path): - gr_warning(translations["input_not_valid"]) - return [None]*4 - - if not os.path.exists(output_dirs): - gr_warning(translations["output_not_valid"]) - return [None]*4 - - if not os.path.exists(output_dirs): os.makedirs(output_dirs) - gr_info(translations["start"].format(start=translations["separator_music"])) - - subprocess.run([ - python, - configs["separate_path"], - "--input_path", input_path, - "--output_dirs", output_dirs, - "--export_format", export_format, - "--model_name", model_name, - "--karaoke_model", karaoke_model, - "--reverb_model", reverb_model, - "--denoise_model", denoise_model, - "--sample_rate", str(sample_rate), - "--shifts", str(shifts), - "--batch_size", str(batch_size), - "--overlap", str(overlap), - "--aggression", str(aggression), - "--hop_length", str(hop_length), - "--window_size", str(window_size), - "--segments_size", str(segments_size), - "--post_process_threshold", str(post_process_threshold), - "--enable_tta", str(enable_tta), - "--enable_denoise", str(enable_denoise), - "--high_end_process", str(high_end_process), - "--enable_post_process", str(enable_post_process), - "--separate_backing", str(separate_backing), - "--separate_reverb", str(separate_reverb), - ]) - - gr_info(translations["success"]) - - filename, _ = os.path.splitext(os.path.basename(input_path)) - output_dirs = os.path.join(output_dirs, filename) - - return [ - os.path.join( - output_dirs, - f"Original_Vocals_No_Reverb.{export_format}" if separate_reverb else f"Original_Vocals.{export_format}" - ), - os.path.join( - output_dirs, - f"Instruments.{export_format}" - ), - os.path.join( - output_dirs, - f"Main_Vocals_No_Reverb.{export_format}" if separate_reverb else f"Main_Vocals.{export_format}" - ) if separate_backing else None, - os.path.join( - output_dirs, - f"Backing_Vocals.{export_format}" - ) if separate_backing else None - ] if os.path.isfile(input_path) else [None]*4 \ No newline at end of file diff --git a/main/app/core/training.py b/main/app/core/training.py deleted file mode 100644 index d4f5718708e65d7304ef5e04160c1159a1df7f62..0000000000000000000000000000000000000000 --- a/main/app/core/training.py +++ /dev/null @@ -1,590 +0,0 @@ -import os -import sys -import time -import shutil -import codecs -import datetime -import threading -import subprocess - -sys.path.append(os.getcwd()) - -from main.tools import huggingface -from main.app.core.ui import gr_info, gr_warning -from main.app.variables import python, translations, configs, file_types, logger - -def getname(path): - return os.path.basename(path).replace(".py", "") - -def if_done(done, p): - while 1: - if p.poll() is None: time.sleep(0.5) - else: break - - done[0] = True - -def log_read(done, name, start_time): - log_file = os.path.join(configs["logs_path"], "app.log") - - def read_logs(): - logs = [] - - with open(log_file, "r", encoding="utf-8") as f: - for line in f: - try: - if ("DEBUG" in line or name not in line or line.strip() == ""): continue - timestamp = datetime.datetime.strptime(line.split("|")[0].strip(), "%Y-%m-%d %H:%M:%S.%f") - if timestamp >= start_time: logs.append(line) - except ValueError: - continue - return "".join(logs) - - while 1: - yield read_logs() - - time.sleep(1) - - if done[0]: - break - - yield read_logs() - -def create_dataset( - input_data, - output_dirs, - skip_seconds, - skip_start_audios, - skip_end_audios, - separate, - model_name, - reverb_model, - denoise_model, - sample_rate, - shifts, - batch_size, - overlap, - aggression, - hop_length, - window_size, - segments_size, - post_process_threshold, - enable_tta, - enable_denoise, - high_end_process, - enable_post_process, - separate_reverb, - clean_dataset, - clean_strength -): - gr_info(translations["start"].format(start=translations["create"])) - start_time = datetime.datetime.now() - - p = subprocess.Popen([ - python, - configs["create_dataset_path"], - "--input_data", input_data, - "--output_dirs", output_dirs, - "--skip_seconds", str(skip_seconds), - "--skip_start_audios", str(skip_start_audios), - "--skip_end_audios", str(skip_end_audios), - "--separate", str(separate), - "--model_name", model_name, - "--reverb_model", reverb_model, - "--denoise_model", denoise_model, - "--sample_rate", str(sample_rate), - "--shifts", str(shifts), - "--batch_size", str(batch_size), - "--overlap", str(overlap), - "--aggression", str(aggression), - "--hop_length", str(hop_length), - "--window_size", str(window_size), - "--segments_size", str(segments_size), - "--post_process_threshold", str(post_process_threshold), - "--enable_tta", str(enable_tta), - "--enable_denoise", str(enable_denoise), - "--high_end_process", str(high_end_process), - "--enable_post_process", str(enable_post_process), - "--separate_reverb", str(separate_reverb), - "--clean_dataset", str(clean_dataset), - "--clean_strength", str(clean_strength), - ]) - - done = [False] - - threading.Thread(target=if_done, args=(done, p)).start() - - for log in log_read(done, getname(configs["create_dataset_path"]), start_time): - yield log - -def create_reference( - audio_path, - reference_name, - pitch_guidance, - use_energy, - version, - embedder_model, - embedders_mode, - f0_method, - predictor_onnx, - f0_up_key, - filter_radius, - f0_autotune, - f0_autotune_strength, - proposal_pitch, - proposal_pitch_threshold, - alpha=0.5, - embedders_mix = False, - embedders_mix_layers = 9, - embedders_mix_ratio = 0.5 -): - gr_info(translations["start"].format(start=translations["create_reference"])) - start_time = datetime.datetime.now() - - p = subprocess.Popen([ - python, - configs["create_reference_path"], - "--audio_path", audio_path, - "--reference_name", reference_name, - "--pitch_guidance", str(pitch_guidance), - "--use_energy", str(use_energy), - "--version", version, - "--embedder_model", embedder_model, - "--embedders_mode", embedders_mode, - "--f0_method", f0_method, - "--predictor_onnx", str(predictor_onnx), - "--f0_up_key", str(f0_up_key), - "--filter_radius", str(filter_radius), - "--f0_autotune", str(f0_autotune), - "--f0_autotune_strength", str(f0_autotune_strength), - "--proposal_pitch", str(proposal_pitch), - "--proposal_pitch_threshold", str(proposal_pitch_threshold), - "--alpha", str(alpha), - "--embedders_mix", str(embedders_mix), - "--embedders_mix_layers", str(embedders_mix_layers), - "--embedders_mix_ratio", str(embedders_mix_ratio) - ]) - - done = [False] - - threading.Thread(target=if_done, args=(done, p)).start() - - for log in log_read(done, getname(configs["create_reference_path"]), start_time): - yield log - -def preprocess( - model_name, - sample_rate, - cpu_core, - cut_preprocess, - process_effects, - dataset, - clean_dataset, - clean_strength, - chunk_len=3.0, - overlap_len=0.3, - normalization_mode="none", - architecture="RVC" -): - sr = int(float(sample_rate.rstrip("k")) * 1000) - - if not model_name: return gr_warning(translations["provide_name"]) - - if configs.get("check_data", False): - try: - found = False - if os.path.exists(dataset): - for root, _, files in os.walk(dataset): - for f in files: - if f.lower().endswith(tuple(file_types)) and os.path.getsize(os.path.join(root, f)) > 0: - found = True - break - - if found: break - if not found: return gr_warning(translations["not_found_data"]) - except Exception: - return gr_warning(translations["not_found_data"]) - - start_time = datetime.datetime.now() - model_dir = os.path.join(configs["logs_path"], model_name) - if os.path.exists(model_dir): shutil.rmtree(model_dir, ignore_errors=True) - - p = subprocess.Popen([ - python, - configs["preprocess_path"], - "--model_name", model_name, - "--dataset_path", dataset, - "--sample_rate", str(sr), - "--cpu_cores", str(cpu_core), - "--cut_preprocess", cut_preprocess, - "--process_effects", str(process_effects), - "--clean_dataset", str(clean_dataset), - "--clean_strength", str(clean_strength), - "--chunk_len", str(chunk_len), - "--overlap_len", str(overlap_len), - "--normalization_mode", normalization_mode, - "--architecture", architecture - ]) - - done = [False] - - threading.Thread(target=if_done, args=(done, p)).start() - os.makedirs(model_dir, exist_ok=True) - - for log in log_read(done, getname(configs["preprocess_path"]), start_time): - yield log - -def extract( - model_name, - version, - method, - pitch_guidance, - hop_length, - cpu_cores, - gpu, - sample_rate, - embedders, - custom_embedders, - predictor_onnx, - embedders_mode, - f0_autotune, - f0_autotune_strength, - hybrid_method, - rms_extract, - alpha=0.5, - include_mutes=2, - embedders_mix = False, - embedders_mix_layers = 9, - embedders_mix_ratio = 0.5, - architecture = "RVC" -): - f0_method, embedder_model = ( - method if method != "hybrid" else hybrid_method, - embedders if embedders != "custom" else custom_embedders - ) - - sr = int(float(sample_rate.rstrip("k")) * 1000) - - if not model_name: return gr_warning(translations["provide_name"]) - model_dir = os.path.join(configs["logs_path"], model_name) - - if configs.get("check_data", False): - try: - if not any( - os.path.isfile(os.path.join(model_dir, "sliced_audios", f)) - for f in os.listdir(os.path.join(model_dir, "sliced_audios")) - ) or not any( - os.path.isfile(os.path.join(model_dir, "sliced_audios_16k", f)) - for f in os.listdir(os.path.join(model_dir, "sliced_audios_16k")) - ): - return gr_warning(translations["not_found_data_preprocess"]) - except: - return gr_warning(translations["not_found_data_preprocess"]) - - start_time = datetime.datetime.now() - - p = subprocess.Popen([ - python, - configs["extract_path"], - "--model_name", model_name, - "--rvc_version", version, - "--f0_method", f0_method, - "--pitch_guidance", str(pitch_guidance), - "--hop_length", str(hop_length), - "--cpu_cores", str(cpu_cores), - "--gpu", str(gpu), - "--sample_rate", str(sr), - "--embedder_model", embedder_model, - "--predictor_onnx", str(predictor_onnx), - "--embedders_mode", embedders_mode, - "--f0_autotune", str(f0_autotune), - "--f0_autotune_strength", str(f0_autotune_strength), - "--rms_extract", str(rms_extract), - "--alpha", str(alpha), - "--include_mutes", str(include_mutes), - "--embedders_mix", str(embedders_mix), - "--embedders_mix_layers", str(embedders_mix_layers), - "--embedders_mix_ratio", str(embedders_mix_ratio), - "--architecture", architecture - ]) - - done = [False] - - threading.Thread(target=if_done, args=(done, p)).start() - os.makedirs(model_dir, exist_ok=True) - - for log in log_read(done, getname(configs["extract_path"]), start_time): - yield log - -def create_index( - model_name, - rvc_version, - index_algorithm, - nprobe=1 -): - if not model_name: return gr_warning(translations["provide_name"]) - model_dir = os.path.join(configs["logs_path"], model_name) - - if configs.get("check_data", False): - try: - if not any( - os.path.isfile(os.path.join(model_dir, f"{rvc_version}_extracted", f)) - for f in os.listdir(os.path.join(model_dir, f"{rvc_version}_extracted")) - ): - return gr_warning(translations["not_found_data_extract"]) - except: - return gr_warning(translations["not_found_data_extract"]) - - start_time = datetime.datetime.now() - - p = subprocess.Popen([ - python, - configs["create_index_path"], - "--model_name", model_name, - "--rvc_version", rvc_version, - "--index_algorithm", index_algorithm, - "--nprobe", str(nprobe) - ]) - - done = [False] - - threading.Thread(target=if_done, args=(done, p)).start() - os.makedirs(model_dir, exist_ok=True) - - for log in log_read(done, getname(configs["create_index_path"]), start_time): - yield log - -def training( - model_name, - rvc_version, - save_every_epoch, - save_only_latest, - save_every_weights, - total_epoch, - sample_rate, - batch_size, - gpu, - pitch_guidance, - not_pretrain, - custom_pretrained, - pretrain_g, - pretrain_d, - detector, - threshold, - clean_up, - cache, - model_author, - vocoder, - checkpointing, - deterministic, - benchmark, - optimizer, - energy_use, - custom_reference=False, - reference_name="", - multiscale_mel_loss=False, - embedders="hubert_base", - custom_embedders=None, - cosine_annealing_lr=False, - architecture="RVC" -): - if architecture == "SVC": - pitch_guidance = True - energy_use = False - - sr = int(float(sample_rate.rstrip("k")) * 1000) - if not model_name: return gr_warning(translations["provide_name"]) - - model_dir = os.path.join(configs["logs_path"], model_name) - if os.path.exists(os.path.join(model_dir, "train_pid.txt")): - os.remove(os.path.join(model_dir, "train_pid.txt")) - - if configs.get("check_data", False): - try: - if not any( - os.path.isfile(os.path.join(model_dir, f"{rvc_version}_extracted", f)) - for f in os.listdir(os.path.join(model_dir, f"{rvc_version}_extracted")) - ): - return gr_warning(translations["not_found_data_extract"]) - except: - return gr_warning(translations["not_found_data_extract"]) - - if not not_pretrain: - if not custom_pretrained: - pretrain_dir = ( - configs["pretrained_v2_path"] - if rvc_version == 'v2' else - configs["pretrained_v1_path"] - ) - - download_version = codecs.decode( - f"uggcf://uhttvatsnpr.pb/NauC/Ivrganzrfr-EIP-Cebwrpg/erfbyir/znva/cergenvarq_", - "rot13" - ) + f"{rvc_version}/" - - pretrained_selector = { - True: { - 24000: ("f0G24k.pth", "f0D24k.pth"), - 32000: ("f0G32k.pth", "f0D32k.pth"), - 40000: ("f0G40k.pth", "f0D40k.pth"), - 44100: ("f0G44k.pth", "f0D44k.pth"), - 48000: ("f0G48k.pth", "f0D48k.pth") - }, - False: { - 24000: ("G24k.pth", "D24k.pth"), - 32000: ("G32k.pth", "D32k.pth"), - 40000: ("G40k.pth", "D40k.pth"), - 44100: ("G44k.pth", "D44k.pth"), - 48000: ("G48k.pth", "D48k.pth") - } - } - - pg2, pd2 = "", "" - pg, pd = pretrained_selector[pitch_guidance][sr] - - if energy_use: pg2, pd2 = pg2 + "ENERGY_", pd2 + "ENERGY_" - if vocoder != 'Default': pg2, pd2 = pg2 + vocoder + "_", pd2 + vocoder + "_" - if embedders not in ["hubert_base", "contentvec_base"]: pg2, pd2 = pg2 + embedders + "_", pd2 + embedders + "_" - if architecture != "RVC": pg2, pd2 = architecture + "_" + pg2, architecture + "_" + pd2 - - pg2, pd2 = pg2 + pg, pd2 + pd - logger.debug("PG: " + pg2 + " PD: " + pd2) - - pretrained_G, pretrained_D = ( - os.path.join( - pretrain_dir, - pg2 - ), - os.path.join( - pretrain_dir, - pd2 - ) - ) - - try: - if not os.path.exists(pretrained_G): - gr_info(translations["download_pretrained"].format(dg="G", rvc_version=rvc_version)) - huggingface.HF_download_file( - "".join( - [ - download_version, - pg2 - ] - ), - os.path.join( - pretrain_dir, - pg2 - ) - ) - - if not os.path.exists(pretrained_D): - gr_info(translations["download_pretrained"].format(dg="D", rvc_version=rvc_version)) - huggingface.HF_download_file( - "".join( - [ - download_version, - pd2 - ] - ), - os.path.join( - pretrain_dir, - pd2 - ) - ) - except: - gr_warning(translations["not_use_pretrain_error_download"]) - pretrained_G = pretrained_D = None - else: - if not pretrain_g: - return gr_warning(translations["provide_pretrained"].format(dg="G")) - - if not pretrain_d: - return gr_warning(translations["provide_pretrained"].format(dg="D")) - - pg2, pd2 = pretrain_g, pretrain_d - pretrained_G, pretrained_D = ( - os.path.join(configs["pretrained_custom_path"], pg2) if not os.path.exists(pg2) else pg2, - os.path.join(configs["pretrained_custom_path"], pd2) if not os.path.exists(pd2) else pd2 - ) - - if not os.path.exists(pretrained_G): - return gr_warning(translations["not_found_pretrain"].format(dg="G")) - - if not os.path.exists(pretrained_D): - return gr_warning(translations["not_found_pretrain"].format(dg="D")) - else: - pretrained_G = pretrained_D = None - gr_warning(translations["not_use_pretrain"]) - - if custom_reference: - embedder_model = embedders if embedders != "custom" else custom_embedders - - reference_path = os.path.join( - configs["reference_path"], - "".join([ - reference_name, - "_", - rvc_version, - "_", - embedder_model, - "_", - str(pitch_guidance), - "_", - str(energy_use) - ]) - ) - - if not os.path.exists(reference_path): - gr_warning(translations["not_found_reference"]) - - custom_reference = False - reference_path = None - else: reference_path = None - - start_time = datetime.datetime.now() - - p = subprocess.Popen([ - python, - configs["train_path"], - "--model_name", model_name, - "--rvc_version", rvc_version, - "--save_every_epoch", str(save_every_epoch), - "--save_only_latest", str(save_only_latest), - "--save_every_weights", str(save_every_weights), - "--total_epoch", str(total_epoch), - "--batch_size", str(batch_size), - "--gpu", str(gpu), - "--pitch_guidance", str(pitch_guidance), - "--overtraining_detector", str(detector), - "--overtraining_threshold", str(threshold), - "--cleanup", str(clean_up), - "--cache_data_in_gpu", str(cache), - "--g_pretrained_path", str(pretrained_G), - "--d_pretrained_path", str(pretrained_D), - "--model_author", model_author, - "--vocoder", vocoder, - "--checkpointing", str(checkpointing), - "--deterministic", str(deterministic), - "--benchmark", str(benchmark), - "--optimizer", optimizer, - "--energy_use", str(energy_use), - "--use_custom_reference", str(custom_reference), - "--reference_path", str(reference_path), - "--multiscale_mel_loss", str(multiscale_mel_loss), - "--use_cosine_annealing_lr", str(cosine_annealing_lr), - "--architecture", architecture - ]) - - done = [False] - - with open(os.path.join(model_dir, "train_pid.txt"), "w") as pid_file: - pid_file.write(str(p.pid)) - - threading.Thread(target=if_done, args=(done, p)).start() - - for log in log_read(done, getname(configs["train_path"]), start_time): - lines = log.splitlines() - - if len(lines) > 50: - log = "\n".join(lines[-50:]) - - yield log \ No newline at end of file diff --git a/main/app/core/tts.py b/main/app/core/tts.py deleted file mode 100644 index 7ea4d78100dd44adf6e591cc806a30b20a5f7a99..0000000000000000000000000000000000000000 --- a/main/app/core/tts.py +++ /dev/null @@ -1,203 +0,0 @@ -import os -import sys -import pysrt -import codecs -import asyncio -import requests -import tempfile - -sys.path.append(os.getcwd()) - -from main.app.variables import translations -from main.app.core.ui import gr_info, gr_warning, gr_error - -def synthesize_tts( - prompt, - voice, - speed, - output, - pitch, - google -): - if not google: - from edge_tts import Communicate - - asyncio.run( - Communicate( - text=prompt, - voice=voice, - rate=f"+{speed}%" if speed >= 0 else f"{speed}%", - pitch=f"+{pitch}Hz" if pitch >= 0 else f"{pitch}Hz" - ).save(output) - ) - else: - response = requests.get( - codecs.decode( - "uggcf://genafyngr.tbbtyr.pbz/genafyngr_ggf", - "rot13" - ), - params={ - "ie": "UTF-8", - "q": prompt, - "tl": voice, - "ttsspeed": max(0.24, min(4.0, 1 + speed / 100)), - "client": "tw-ob" - }, - headers={ - "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36" - }, - timeout=15 - ) - - if response.status_code == 200: - with open(output, "wb") as f: - f.write(response.content) - - if pitch != 0 or speed != 0: - import librosa - import soundfile as sf - - y, sr = librosa.load(output, sr=None) - - if pitch != 0: - y = librosa.effects.pitch_shift( - y, - sr=sr, - n_steps=pitch - ) - - rate = max(0.01, 1 + speed / 100) - if rate != 1.0: - y = librosa.effects.time_stretch( - y, - rate=rate - ) - - sf.write( - file=output, - data=y, - samplerate=sr, - format=os.path.splitext( - os.path.basename(output) - )[-1].lower().replace('.', '') - ) - else: gr_error(f"{response.status_code}, {response.text}") - -def srt_tts( - srt_file, - out_file, - voice, - rate = 0, - sr = 24000, - google = False -): - import librosa - import numpy as np - import soundfile as sf - - def time_stretch(y, sr, target_duration): - rate = (len(y) / sr) / target_duration - - if rate != 1.0: - y = librosa.effects.time_stretch( - y=y.astype(np.float32), - rate=rate - ) - - n_target = int(round(target_duration * sr)) - return np.pad(y, (0, n_target - len(y))) if len(y) < n_target else y[:n_target] - - def pysrttime_to_seconds(t): - return (t.hours * 60 + t.minutes) * 60 + t.seconds + t.milliseconds / 1000 - - subs = pysrt.open(srt_file) - if not subs: raise ValueError(translations["srt"]) - - final_audio = np.zeros( - int(round(pysrttime_to_seconds(subs[-1].end) * sr)), - dtype=np.float32 - ) - - with tempfile.TemporaryDirectory() as tempdir: - for idx, seg in enumerate(subs): - wav_path = os.path.join(tempdir, f"seg_{idx}.wav") - - synthesize_tts( - " ".join(seg.text.splitlines()), - voice, - 0, - wav_path, - rate, - google - ) - - audio, file_sr = sf.read(wav_path, dtype=np.float32) - if file_sr != sr: audio = librosa.resample(audio, orig_sr=file_sr, target_sr=sr, res_type="soxr_vhq") - - adjusted = time_stretch( - audio, - sr, - pysrttime_to_seconds(seg.duration) - ) - - start_sample = int(round(pysrttime_to_seconds(seg.start) * sr)) - end_sample = start_sample + adjusted.shape[0] - - if end_sample > final_audio.shape[0]: - adjusted = adjusted[: final_audio.shape[0] - start_sample] - end_sample = final_audio.shape[0] - - final_audio[start_sample:end_sample] += adjusted - - sf.write(out_file, final_audio, sr) - -def TTS( - prompt, - voice, - speed, - output, - pitch, - google, - srt_input -): - if not srt_input: srt_input = "" - - if not prompt and not srt_input.endswith(".srt"): - gr_warning(translations["enter_the_text"]) - return None - - if not voice: - gr_warning(translations["choose_voice"]) - return None - - if not output: - gr_warning(translations["output_not_valid"]) - return None - - if os.path.isdir(output): output = os.path.join(output, f"tts.wav") - gr_info(translations["convert"].format(name=translations["text"])) - - output_dir = os.path.dirname(output) or output - if not os.path.exists(output_dir): os.makedirs(output_dir, exist_ok=True) - - if srt_input.endswith(".srt"): - srt_tts( - srt_input, - output, - voice, - 0, - 24000, - google - ) - else: - synthesize_tts( - prompt, - voice, - speed, - output, - pitch, - google - ) - - gr_info(translations["success"]) - return output \ No newline at end of file diff --git a/main/app/core/ui.py b/main/app/core/ui.py deleted file mode 100644 index 5bc8b5a25e25120e639c241cd6f5b7f42b8946d1..0000000000000000000000000000000000000000 --- a/main/app/core/ui.py +++ /dev/null @@ -1,814 +0,0 @@ -import os -import re -import sys -import json -import torch -import shutil - -import gradio as gr - -sys.path.append(os.getcwd()) - -import main.library.audio.sounddevice as sd - -from main.library.backends import directml, opencl -from main.inference.realtime.audio import audio_device - -from main.app.variables import ( - logger, - config, - configs, - edgetts, - method_f0, - vr_models, - mdx_models, - file_types, - configs_json, - translations, - whisper_model, - demucs_models, - method_f0_full, - gradio_version, - embedders_model, - google_tts_voice -) - -input_channels_map, output_channels_map = audio_device() - -def gr_info(message): - gr.Info(message, duration=2) - logger.info(message) - -def gr_warning(message): - gr.Warning(message, duration=2) - logger.warning(message) - -def gr_error(message): - gr.Error(message=message, duration=6) - logger.error(message) - -def get_gpu_info(): - ngpu = torch.cuda.device_count() - gpu_infos = [ - ( - f"{i}: {torch.cuda.get_device_name(i)} " + - f"({int(torch.cuda.get_device_properties(i).total_memory / (1024**3) + 0.4)} GB)" - ) - for i in range(ngpu) - if config.device.startswith("cuda") or ngpu != 0 - ] - - if len(gpu_infos) == 0 and config.device.startswith("xpu"): - ngpu = torch.xpu.device_count() - gpu_infos = [ - ( - f"{i}: {torch.xpu.get_device_name(i)} " + - f"({int(torch.xpu.get_device_properties(i).total_memory / (1024**3) + 0.4)} GB)" - ) - for i in range(ngpu) - if config.device.startswith("xpu") or ngpu != 0 - ] - - if len(gpu_infos) == 0: - if config.device.startswith("privateuseone"): - ngpu = directml.device_count() - - gpu_infos = [ - f"{i}: {directml.device_name(i)}" - for i in range(ngpu) - if config.device.startswith("privateuseone") or ngpu != 0 - ] - elif config.device.startswith("ocl"): - ngpu = opencl.device_count() - - gpu_infos = [ - f"{i}: {opencl.device_name(i)}" - for i in range(ngpu) - if config.device.startswith("ocl") or ngpu != 0 - ] - else: - ngpu = 0 - gpu_infos = [] - - return ( - ( - "\n".join(gpu_infos) - if len(gpu_infos) > 0 and not config.cpu_mode else - translations["no_support_gpu"] - ), - ngpu - ) - -def gpu_number_str(): - if config.cpu_mode: return "-" - ngpu = torch.cuda.device_count() - - if ngpu == 0 and config.device.startswith("xpu"): - ngpu = torch.xpu.device_count() - if ngpu != 0: return "0" - - if ngpu == 0: - ngpu = ( - directml.device_count() - if config.device.startswith("privateuseone") else - opencl.device_count() - ) - if ngpu != 0: return "0" - - return str("-".join(map(str, range(ngpu))) if config.device.startswith(("cuda", "xpu", "privateuseone", "ocl")) else "-") - -def change_f0_choices(): - f0_file = sorted([ - os.path.abspath(os.path.join(root, f)) - for root, _, files in os.walk(configs["f0_path"]) - for f in files - if f.endswith(".txt") - ]) - - return { - "value": f0_file[0] if len(f0_file) >= 1 else "", - "choices": f0_file, - "__type__": "update" - } - -def change_audios_choices(input_audio): - audios = sorted([ - os.path.abspath(os.path.join(root, f)) - for root, _, files in os.walk(configs["audios_path"]) - for f in files if os.path.splitext(f)[1].lower() in file_types and not f.startswith(("Convert", "output")) - ]) - - return { - "value": input_audio if input_audio != "" else (audios[0] if len(audios) >= 1 else ""), - "choices": audios, - "__type__": "update" - } - -def change_reference_choices(): - reference = sorted([ - re.sub(r'_v\d+_(?:[A-Za-z0-9_]+?)_(True|False)_(True|False)$', '', name) - for name in os.listdir(configs["reference_path"]) - if ( - os.path.exists(os.path.join(configs["reference_path"], name)) and - os.path.isdir(os.path.join(configs["reference_path"], name)) - ) - ]) - - return { - "value": reference[0] if len(reference) >= 1 else "", - "choices": reference, - "__type__": "update" - } - -def change_models_choices(): - model, index = ( - sorted([ - model - for model in os.listdir(configs["weights_path"]) - if model.endswith((".pth", ".onnx")) and not model.startswith("G_") and not model.startswith("D_") - ]), - sorted([ - os.path.join(root, name) - for root, _, files in os.walk(configs["logs_path"], topdown=False) - for name in files if name.endswith(".index") and "trained" not in name - ]) - ) - - return [ - { - "value": model[0] if len(model) >= 1 else "", - "choices": model, - "__type__": "update" - }, - { - "value": index[0] if len(index) >= 1 else "", - "choices": index, - "__type__": "update" - } - ] - -def change_pretrained_choices(): - pretrainD = sorted([ - model for model in os.listdir(configs["pretrained_custom_path"]) - if model.endswith(".pth") and "D" in model - ]) - - pretrainG = sorted([ - model for model in os.listdir(configs["pretrained_custom_path"]) - if model.endswith(".pth") and "G" in model - ]) - - return [ - { - "choices": pretrainD, - "value": pretrainD[0] if len(pretrainD) >= 1 else "", - "__type__": "update" - }, - { - "choices": pretrainG, - "value": pretrainG[0] if len(pretrainG) >= 1 else "", - "__type__": "update" - } - ] - -def change_preset_choices(): - return { - "value": "", - "choices": sorted([ - f for f in os.listdir(configs["presets_path"]) - if f.endswith(".conversion.json") - ]), - "__type__": "update" - } - -def change_effect_preset_choices(): - return { - "value": "", - "choices": sorted([ - f for f in os.listdir(configs["presets_path"]) - if f.endswith(".effect.json") - ]), - "__type__": "update" - } - -def change_realtime_preset_choices(): - return { - "value": "", - "choices": sorted([ - f for f in os.listdir(configs["presets_path"]) - if f.endswith(".realtime.json") - ]), - "__type__": "update" - } - -def change_tts_voice_choices(google): - return { - "choices": google_tts_voice if google else edgetts, - "value": google_tts_voice[0] if google else edgetts[0], - "__type__": "update" - } - -def change_backing_choices( - backing, - merge -): - if backing or merge: - return { - "value": False, - "interactive": False, - "__type__": "update" - } - elif not backing or not merge: - return { - "interactive": True, - "__type__": "update" - } - else: - gr_warning(translations["option_not_valid"]) - -def change_download_choices(select): - selects = [False]*10 - - if select == translations["download_url"]: - selects[0] = selects[1] = selects[2] = True - elif select == translations["download_from_csv"]: - selects[3] = selects[4] = True - elif select == translations["search_models"]: - selects[5] = selects[6] = True - elif select == translations["upload"]: - selects[9] = True - else: - gr_warning(translations["option_not_valid"]) - - return [ - { - "visible": selects[i] if gradio_version else (selects[i] or "hidden"), - "__type__": "update" - } - for i in range(len(selects)) - ] - -def change_download_pretrained_choices(select): - selects = [False]*7 - - if select == translations["download_url"]: - selects[0] = selects[1] = selects[2] = True - elif select == translations["list_model"]: - selects[3] = selects[4] = selects[5] = True - elif select == translations["upload"]: - selects[6] = True - else: - gr_warning(translations["option_not_valid"]) - - return [ - { - "visible": selects[i] if gradio_version else (selects[i] or "hidden"), - "__type__": "update" - } - for i in range(len(selects)) - ] - -def get_index(model): - model = os.path.basename(model).split("_")[0] - - return { - "value": next(( - f for f in [ - os.path.join(root, name) - for root, _, files in os.walk(configs["logs_path"], topdown=False) - for name in files - if name.endswith(".index") and "trained" not in name - ] - if model.split(".")[0] in f - ), ""), - "__type__": "update" - } if model else None - -def index_strength_show(index): - index_strength_visible = ( - index != "" and - index != None and - os.path.exists(index) and - os.path.isfile(index) - ) - - return [ - { - "visible": index_strength_visible if gradio_version else (index_strength_visible or "hidden"), - "value": 0.5, - "__type__": "update" - }, - { - "visible": index_strength_visible if gradio_version else (index_strength_visible or "hidden"), - "value": 9, - "__type__": "update" - } - ] - -def hoplength_show( - method, - hybrid_method=None -): - visible = False - - for m in [ - "mangio-crepe", - "yin", - "piptrack", - "mangio-penn" - ]: - if m in method: visible = True - - if ( - hybrid_method is not None and - m in hybrid_method - ): - visible = True - - if visible: break - else: visible = False - - return { - "visible": visible if gradio_version else (visible or "hidden"), - "__type__": "update" - } - -def visible(value): - return { - "visible": value if gradio_version else (value or "hidden"), - "__type__": "update" - } - -def visibleFalse(value): - return { - "visible": value if gradio_version else (value or "hidden"), - "value": False, - "__type__": "update" - } - -def valueFalse_interactive(value): - return { - "value": False, - "interactive": value, - "__type__": "update" - } - -def interactive(value): - return { - "interactive": value, - "__type__": "update" - } - -def valueEmpty_visible1(value): - return { - "value": "", - "visible": value if gradio_version else (value or "hidden"), - "__type__": "update" - } - -def pitch_guidance_lock(vocoders): - return { - "value": True, - "interactive": vocoders == "Default", - "__type__": "update" - } - -def vocoders_lock(pitch): - if pitch: - return gr.update(interactive=True) - - return gr.update( - value="Default", - interactive=False - ) - -def unlock_f0(value): - return { - "choices": method_f0_full if value else method_f0, - "value": "rmvpe", - "__type__": "update" - } - -def unlock_vocoder(value, vocoder): - return { - "value": vocoder if value == "v2" else "Default", - "interactive": value == "v2", - "__type__": "update" - } - -def unlock_version(value, vocoder): - return { - "value": "v2" if vocoder == "Default" else value, - "interactive": vocoder == "Default", - "__type__": "update" - } - -def change_embedders_mode(value): - if value == "whisper": - return { - "value": whisper_model[0], - "choices": whisper_model, - "__type__": "update" - } - else: - return { - "value": embedders_model[0], - "choices": embedders_model, - "__type__": "update" - } - -def change_fp(fp, bf16, tf32): - global config - - fp16 = fp == "fp16" - - if fp16 and not config.allow_is_half: - gr_warning(translations["fp16_not_support"]) - return "fp32" - else: - gr_info(translations["start_update_precision"]) - - configs = json.load(open(configs_json, "r")) - configs["fp16"] = config.is_half = fp16 - configs["brain"] = config.brain = bf16 - configs["tf32"] = config.tf32 = tf32 - - with open(configs_json, "w") as f: - json.dump(configs, f, indent=4) - - gr_info(translations["success"]) - return "fp16" if fp16 else "fp32" - -def process_output(file_path): - if config.configs.get("delete_exists_file", True): - if os.path.exists(file_path) and os.path.isfile(file_path): os.remove(file_path) - return file_path - else: - if not os.path.exists(file_path): return file_path - file = os.path.splitext(os.path.basename(file_path)) - - index = 1 - while 1: - file_path = os.path.join( - os.path.dirname(file_path), - f"{file[0]}_{index}{file[1]}" - ) - - if not os.path.exists(file_path): return file_path - index += 1 - -def shutil_move(input_path, output_path): - output_path = ( - os.path.join(output_path, os.path.basename(input_path)) - if os.path.isdir(output_path) else - output_path - ) - - return ( - shutil.move(input_path, process_output(output_path)) - if os.path.exists(output_path) else - shutil.move(input_path, output_path) - ) - -def separate_change( - model_name, - karaoke_model, - reverb_model, - enable_post_process, - separate_backing, - separate_reverb, - enable_denoise -): - model_type = ( - "vr" if model_name in list(vr_models.keys()) else "mdx" - if model_name in list(mdx_models.keys()) else "demucs" - if model_name in list(demucs_models.keys()) else "" - ) - - karaoke_type = ("vr" if karaoke_model.startswith("VR") else "mdx") if separate_backing else None - reverb_type = ("vr" if not reverb_model.startswith("MDX") else "mdx") if separate_reverb else None - - all_types = {model_type, karaoke_type, reverb_type} - - is_vr = "vr" in all_types - is_mdx = "mdx" in all_types - is_demucs = "demucs" in all_types - - return [ - visible(separate_backing), - visible(separate_reverb), - visible(is_mdx or is_demucs), - visible(is_mdx or is_demucs), - visible(is_mdx), - visible(is_mdx or is_vr), - visible(is_demucs), - visible(is_vr), - visible(is_vr), - visible(is_vr and enable_post_process), - visible(is_vr and enable_denoise), - valueFalse_interactive(is_vr), - valueFalse_interactive(is_vr), - interactive(is_vr) if is_vr else valueFalse_interactive(is_vr) - ] - -def create_dataset_change( - model_name, - reverb_model, - enable_post_process, - separate_reverb, - enable_denoise -): - model_type = ( - "vr" if model_name in list(vr_models.keys()) else "mdx" - if model_name in list(mdx_models.keys()) else "demucs" - if model_name in list(demucs_models.keys()) else "" - ) - - reverb_type = ("vr" if not reverb_model.startswith("MDX") else "mdx") if separate_reverb else None - all_types = {model_type, reverb_type} - - is_vr = "vr" in all_types - is_mdx = "mdx" in all_types - is_demucs = "demucs" in all_types - - return [ - visible(separate_reverb), - visible(is_mdx or is_demucs), - visible(is_mdx or is_demucs), - visible(is_mdx), - visible(is_mdx or is_vr), - visible(is_demucs), - visible(is_vr), - visible(is_vr), - visible(is_vr and enable_post_process), - visible(is_vr and enable_denoise), - valueFalse_interactive(is_vr), - valueFalse_interactive(is_vr), - interactive(is_vr) if is_vr else valueFalse_interactive(is_vr) - ] - -def update_audio_device(input_device, output_device, monitor_device, monitor): - input_is_asio = "ASIO" in input_device if input_device else (False if gradio_version else "hidden") - output_is_asio = "ASIO" in output_device if output_device else (False if gradio_version else "hidden") - monitor_is_asio = "ASIO" in monitor_device if monitor_device else (False if gradio_version else "hidden") - - if input_is_asio or output_is_asio or monitor_is_asio: - sd.terminate() - sd.initialize() - - try: - input_max_ch = input_channels_map.get(input_device, [])[1] - output_max_ch = output_channels_map.get(output_device, [])[1] - monitor_max_ch = output_channels_map.get(monitor_device, [])[1] if monitor else 128 - except: - input_max_ch = output_max_ch = monitor_max_ch = 2 - else: input_max_ch = output_max_ch = monitor_max_ch = 2 - - return [ - visible(monitor), - visible(monitor), - visible( - input_is_asio or output_is_asio or monitor_is_asio - ), - gr.update( - visible=input_is_asio, - maximum=input_max_ch - 1 - ), - gr.update( - visible=output_is_asio, - maximum=output_max_ch - 1 - ), - gr.update( - visible=monitor_is_asio, - maximum=monitor_max_ch - 1 - ), - gr.update( - value=resolve_sample_rate(input_channels_map[input_device][0]) if input_is_asio else 48000 - ), - gr.update( - value=resolve_sample_rate(output_channels_map[output_device][0]) if output_is_asio else 48000 - ), - gr.update( - visible=monitor, - value=resolve_sample_rate(output_channels_map[monitor_device][0]) if monitor_is_asio else 48000 - ), - gr.update( - visible=output_is_asio, - value=True - ), - gr.update( - visible=monitor_is_asio, - value=True - ), - ] - -def change_audio_device_choices(): - global input_channels_map, output_channels_map - - sd.terminate() - sd.initialize() - - input_channels_map, output_channels_map = audio_device() - input_channels_map_list, output_channels_map_list = list(input_channels_map.keys()), list(output_channels_map.keys()) - - return [ - { - "value": input_channels_map_list[0] if len(input_channels_map_list) >= 1 else "", - "choices": input_channels_map_list, - "__type__": "update" - }, - { - "value": output_channels_map_list[0] if len(output_channels_map_list) >= 1 else "", - "choices": output_channels_map_list, - "__type__": "update" - }, - { - "value": output_channels_map_list[0] if len(output_channels_map_list) >= 1 else "", - "choices": output_channels_map_list, - "__type__": "update" - } - ] - -def resolve_sample_rate(input_device_id): - if configs.get("asio_enabled", False): return 48000 - - try: - device = sd.query_devices(input_device_id) - return int(device["default_samplerate"]) - except Exception: - return 48000 - -def replace_punctuation(filename): - return re.sub(r"_+", "_", re.sub(r"[ \-|]+", "_", filename.translate(str.maketrans("", "", '()[],"\'{}<>?*:/\\')))).strip("_") - -def replace_url(url): - return url.replace("/blob/", "/resolve/").replace("?download=true", "").strip() - -def replace_modelname(modelname): - return replace_punctuation( - modelname.replace(".onnx", "").replace(".pth", "").replace(".index", "").replace(".zip", "") - ) - -def replace_export_format(audio_path, export_format = "wav"): - export_format = f".{export_format}" - - return ( - audio_path - if audio_path.endswith(export_format) else - audio_path.replace(f".{os.path.basename(audio_path).split('.')[-1]}", export_format) - ) - -def update_dropdowns_from_json(data): - if not data: - return [ - gr.update(choices=[], value=None), - gr.update(choices=[], value=None), - gr.update(choices=[], value=None) - ] - - inputs = list(data.get("inputs", {}).keys()) - outputs = list(data.get("outputs", {}).keys()) - - return [ - gr.update( - choices=inputs, - value=inputs[0] if len(inputs) > 0 else None - ), - gr.update( - choices=outputs, - value=outputs[0] if len(outputs) > 0 else None - ), - gr.update( - choices=outputs, - value=outputs[0] if len(outputs) > 0 else None - ), - ] - -def update_button_from_json(data): - if not data: - return [ - gr.update(interactive=True), - gr.update(interactive=False) - ] - - return [ - gr.update(interactive=data.get("start_button", True)), - gr.update(interactive=data.get("stop_button", False)) - ] - -def update_value_from_json(data): - if not data: - return [ - gr.update(), - gr.update(value=None) - ] - - return [ - data.get("button", "Start"), - data.get("path", None), - ] - -def get_speakers_id_and_architecture(model): - model_path = os.path.join(configs["weights_path"], model) if not os.path.exists(model) else model - - if not model or not os.path.exists(model_path) or os.path.isdir(model_path) or not model.endswith((".pth", ".onnx")): - return [ - { - "visible": False if gradio_version else "hidden", - "value": 0, - "choices": [0], - "__type__": "update" - }, - { - "visible": False if gradio_version else "hidden", - "value": 0.4, - "__type__": "update" - } - ] - - try: - if model_path.endswith(".pth"): - model_data = torch.load(model_path, map_location="cpu", weights_only=True) - else: - import onnx - - model_data = None - for prop in onnx.load(model_path).metadata_props: - if prop.key == "model_info": - model_data = json.loads(prop.value) - break - - speakers_id = model_data.get("speakers_id", 1) - speakers_id_visible = speakers_id and speakers_id != 1 - noise_scale_visible = model_data.get("architecture", "RVC") == "SVC" - - return [ - { - "visible": speakers_id_visible if gradio_version else (speakers_id_visible or "hidden"), - "value": 0, - "choices": list(range(speakers_id)), - "__type__": "update" - }, - { - "visible": noise_scale_visible if gradio_version else (noise_scale_visible or "hidden"), - "value": 0.4, - "__type__": "update" - } - ] - except Exception: - return [ - { - "visible": False if gradio_version else "hidden", - "value": 0, - "choices": [0], - "__type__": "update" - }, - { - "visible": False if gradio_version else "hidden", - "value": 0.4, - "__type__": "update" - } - ] - -def run_commands(cmd): - os.system(cmd) \ No newline at end of file diff --git a/main/app/core/utils.py b/main/app/core/utils.py deleted file mode 100644 index 4763440ff86136d4c26a73d18f1d6e3bd41081f1..0000000000000000000000000000000000000000 --- a/main/app/core/utils.py +++ /dev/null @@ -1,123 +0,0 @@ -import os -import sys -import time -import json -import signal -import codecs -import requests -import subprocess - -sys.path.append(os.getcwd()) - -from main.app.core.ui import gr_info, gr_warning -from main.app.variables import translations, configs - -def alive(pid): - try: - if sys.platform == "win32": - subprocess.check_output(["tasklist", "/FI", f"PID eq {pid}"], stderr=subprocess.DEVNULL) - return True - else: - os.kill(pid, 0) - return True - except: - return False - -def pid_kill(pid): - if not alive(pid): return True - - try: - if sys.platform == "win32": subprocess.run(["taskkill", "/PID", str(pid), "/T"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - else: os.kill(pid, signal.SIGINT) - except: - pass - - for _ in range(15): - if not alive(pid): return True - time.sleep(0.1) - - try: - if sys.platform == "win32": subprocess.run(["taskkill", "/PID", str(pid), "/F", "/T"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - else: os.kill(pid, signal.SIGKILL) - except: - pass - - for _ in range(10): - if not alive(pid): return True - time.sleep(0.1) - - return False - -def stop_pid( - pid_file, - model_name=None, - train=False -): - try: - pid_file_path = os.path.join( - "assets", f"{pid_file}.txt" - ) if model_name is None else os.path.join( - configs["logs_path"], model_name, f"{pid_file}.txt" - ) - - if not os.path.exists(pid_file_path): return gr_warning(translations["not_found_pid"]) - else: - with open(pid_file_path, "r") as pid_file: - pids = [int(pid) for pid in pid_file.readlines()] - - for pid in pids: - pid_kill(pid) - - if os.path.exists(pid_file_path): os.remove(pid_file_path) - - pid_file_path = os.path.join(configs["logs_path"], model_name, "config.json") - - if train and os.path.exists(pid_file_path): - with open(pid_file_path, "r") as pid_file: - pid_data = json.load(pid_file) - pids = pid_data.get("process_pids", []) - - with open(pid_file_path, "w") as pid_file: - pid_data.pop("process_pids", None) - - json.dump(pid_data, pid_file, indent=4) - - for pid in pids: - pid_kill(pid) - - gr_info(translations["end_pid"]) - except: - pass - -def google_translate(text, source='auto', target='vi'): - if text == "": return gr_warning(translations["prompt_warning"]) - - try: - import textwrap - - def translate_chunk(chunk): - response = requests.get( - codecs.decode("uggcf://genafyngr.tbbtyrncvf.pbz/genafyngr_n/fvatyr", "rot13"), - params={ - 'client': 'gtx', - 'sl': source, - 'tl': target, - 'dt': 't', - 'q': chunk - } - ) - - return ''.join([i[0] for i in response.json()[0]]) if response.status_code == 200 else chunk - - translated_text = '' - for chunk in textwrap.wrap( - text, - 5000, - break_long_words=False, - break_on_hyphens=False - ): - translated_text += translate_chunk(chunk) - - return translated_text - except: - return text \ No newline at end of file diff --git a/main/app/install.py b/main/app/install.py deleted file mode 100644 index 412cfb4559b39c82b236359e5d3dd3c6e815eea1..0000000000000000000000000000000000000000 --- a/main/app/install.py +++ /dev/null @@ -1,205 +0,0 @@ -import os -import sys -import subprocess - -sys.path.append(os.getcwd()) - -os.makedirs(os.path.join("assets", ".uv"), exist_ok=True) -os.environ["UV_CACHE_DIR"] = os.path.join("assets", ".uv") - -python_dir = os.path.join("runtime", "python.exe") - -def run_command(command): - try: - subprocess.run(command, shell=True, check=True) - except subprocess.CalledProcessError as e: - print(f"Lỗi khi thực thi lệnh: {e}") - -def install_dependencies(): - print("Cài đặt các thư viện cơ bản...") - - run_command(f"{python_dir} -m pip install uv") - run_command(f"{python_dir} -m uv pip install six packaging python-dateutil platformdirs pywin32 wget") - -def install_ffmpeg(): - if not os.path.exists("ffmpeg.exe"): - print("Cài đặt ffmpeg...") - run_command('curl -L -o ffmpeg.exe https://huggingface.co/AnhP/Vietnamese-RVC-Project/resolve/main/ffmpeg/ffmpeg.exe') - - if not os.path.exists("ffprobe.exe"): - print("Cài đặt ffprobe...") - run_command('curl -L -o ffprobe.exe https://huggingface.co/AnhP/Vietnamese-RVC-Project/resolve/main/ffmpeg/ffprobe.exe') - -def remove_onnxruntime(): - if not os.path.exists("requirements.txt"): return - - with open("requirements.txt", "r", encoding="utf-8") as f: - lines = f.readlines() - - with open("requirements.txt", "w", encoding="utf-8") as f: - f.writelines(l for l in lines if "onnxruntime" not in l) - -def install_cpu(): - install_dependencies() - remove_onnxruntime() - - print("Cài đặt các gói thư viện cpu...") - - run_command(f"{python_dir} -m uv pip install torch torchaudio torchvision --index-url https://download.pytorch.org/whl/cpu") - run_command(f"{python_dir} -m uv pip install onnxruntime") - run_command(f"{python_dir} -m uv pip install -r requirements.txt") - -def install_cuda(version = "cu118"): - install_dependencies() - remove_onnxruntime() - - print("Cài đặt các gói thư viện cuda...") - - if version in ("cu118", "cu121"): run_command(f"{python_dir} -m uv pip install numpy==1.26.4 numba==0.61.0") - run_command(f"{python_dir} -m uv pip install torch torchaudio torchvision --index-url https://download.pytorch.org/whl/{version}") - run_command(f"{python_dir} -m uv pip install onnxruntime-gpu" + ("==1.20.1" if version in ("cu118", "cu121") else "")) - run_command(f"{python_dir} -m uv pip install -r requirements.txt") - if version in ("cu128", "cu130"): run_command(f"{python_dir} -m uv pip install faiss-cpu==1.13.2") - -def install_directml(): - install_dependencies() - remove_onnxruntime() - - print("Cài đặt các gói thư viện directml...") - - run_command(f"{python_dir} -m uv pip install numpy==1.26.4 numba==0.61.0") - run_command(f"{python_dir} -m uv pip install torch==2.4.1 torchaudio==2.4.1 torchvision") - run_command(f"{python_dir} -m uv pip install torch-directml==0.2.5.dev240914") - run_command(f"{python_dir} -m uv pip install onnxruntime-directml") - run_command(f"{python_dir} -m uv pip install -r requirements.txt") - -def install_opencl(): - install_dependencies() - remove_onnxruntime() - - print("Cài đặt các gói thư viện opencl...") - - run_command(f"{python_dir} -m uv pip install numpy==1.26.4 numba==0.61.0") - run_command(f"{python_dir} -m uv pip install torch==2.6.0 torchaudio==2.6.0 torchvision") - run_command(f"{python_dir} -m uv pip install https://github.com/artyom-beilis/pytorch_dlprim/releases/download/0.2.0/pytorch_ocl-0.2.0+torch2.6-cp311-none-win_amd64.whl") - run_command(f"{python_dir} -m uv pip install onnxruntime-directml") - run_command(f"{python_dir} -m uv pip install -r requirements.txt") - -def install_xpu(provider = "openvino"): - install_dependencies() - remove_onnxruntime() - - print("Cài đặt các gói thư viện xpu...") - - run_command(f"{python_dir} -m uv pip install torch torchaudio torchvision --index-url https://download.pytorch.org/whl/xpu") - run_command(f"{python_dir} -m uv pip install openvino==2025.4.1") - run_command(f"{python_dir} -m uv pip install triton-windows") - run_command(f"{python_dir} -m uv pip install " + ("onnxruntime-openvino==1.24.1" if provider == "openvino" else "onnxruntime-directml")) - run_command(f"{python_dir} -m uv pip install -r requirements.txt") - run_command(f"{python_dir} -m uv pip install faiss-cpu==1.13.2") - -def install_requirements(): - print("Các phiên bản Backend: \n - 1. CPU (Đối với CPU) / 2. CUDA (Đối với GPU NVIDIA) / 3. DIRECTML (Đối với GPU NVIDIA / AMD / INTEL) / 4. OPENCL (Đối với GPU AMD) / 5. XPU (Đối với GPU INTEL)") - - while 1: - selected = str(input("Hãy chọn phiên bản Backend (Nhập 1-5): ")).strip().upper() - - if selected in ("1", "CPU"): - print("Cài đặt dành cho CPU...") - install_cpu() - break - elif selected in ("2", "CUDA"): - print("Cài đặt dành cho CUDA...") - print("Các phiên bản CUDA: \n Lưu ý: Hãy chọn đúng phiên bản nếu như chọn sai có thể gây ra lỗi suy luận hoặc huấn luyện. \n - 1. Phiên bản 118 dành cho các GPU GTX 10-Series trở lên. \n - 2. Phiên bản 121 dành cho các GPU RTX 20-Series đến 30-Series trở lên. \n - 3. Phiên bản 128 dành cho các GPU RTX 30-Series đến RTX 40-Series trở lên. \n - 4. Phiên bản 130 dành cho các GPU RTX 50-Series trở lên.") - - cuda_selected = None - - while 1: - cuda_selected = str(input("Hãy chọn phiên bản CUDA (Nhập 1-4): ")).strip().upper() - - if cuda_selected in ("1", "118"): - install_cuda("cu118") - break - elif cuda_selected in ("2", "121"): - install_cuda("cu121") - break - elif cuda_selected in ("3", "128"): - install_cuda("cu128") - break - elif cuda_selected in ("4", "130"): - install_cuda("cu130") - break - else: - print("Vui lòng chọn phiên bản CUDA hợp lệ!") - - if cuda_selected in ("cu128", "cu130"): - print("Cài đặt gói phụ thuộc TensorRT (Cực Kỳ Không Khuyên Dùng)") - - while 1: - tensorrt_selected = str(input("Bạn có muốn sử dụng TensorRT không (Y/N): ")).strip().upper() - - if tensorrt_selected in ("Y", "YES"): - run_command(f"{python_dir} -m uv pip install tensorrt") - break - elif tensorrt_selected in ("N", "NO"): - break - else: - print("Vui lòng chọn giá trị hợp lệ!") - - print("Cài đặt gói phụ thuộc Triton (Không Khuyên Dùng)") - - while 1: - triton_selected = str(input("Bạn có muốn sử dụng Triton không (Y/N): ")).strip().upper() - - if triton_selected in ("Y", "YES"): - run_command(f"{python_dir} -m uv pip install triton-windows") - break - elif triton_selected in ("N", "NO"): - break - else: - print("Vui lòng chọn giá trị hợp lệ!") - - break - elif selected in ("3", "DIRECTML"): - print("Cài đặt dành cho DIRECTML...") - install_directml() - break - elif selected in ("4", "OPENCL"): - print("Cài đặt dành cho OPENCL...") - install_opencl() - break - elif selected in ("5", "XPU"): - print("Cài đặt dành cho XPU...") - print("Phiên bản ONNXRUNTIME: 1. DIRECTML (Sử dụng trên mọi GPU) / 2. OPENVINO (Chỉ Intel GPU và tối ưu nhất)") - - while 1: - provider_selected = str(input("Hãy chọn phiên bản ONNXRUNTIME (1-2): ")).strip().upper() - - if provider_selected in ("1", "DIRECTML"): - install_xpu("directml") - break - elif provider_selected in ("2", "OPENVINO"): - install_xpu("openvino") - break - else: - print("Vui lòng chọn giá trị hợp lệ!") - - break - else: - print("Vui lòng chọn phiên bản hợp lệ!") - - print("Hoàn tất quá trình cài đặt thư viện!") - -if __name__ == "__main__": - if not os.path.exists(python_dir): - print("Không tìm thấy thời gian chạy python!") - sys.exit() - - if sys.platform != "win32": - print("Chỉ hỗ trợ trên Windows!") - sys.exit() - - print("Lưu Ý: Hãy cài các phần phụ trợ bên ngoài được đề cập trong phần README.md trước khi tiếp tục!") - - install_requirements() - install_ffmpeg() \ No newline at end of file diff --git a/main/app/parser.py b/main/app/parser.py deleted file mode 100644 index 368894e3bccaa6ef53fa03e091d28eac799795f2..0000000000000000000000000000000000000000 --- a/main/app/parser.py +++ /dev/null @@ -1,407 +0,0 @@ -import os -import sys - -sys.path.append(os.getcwd()) - -try: - argv = sys.argv[1] -except IndexError: - argv = None - -argv_help_is_allows = [ - "--help_audio_effects", - "--help_convert", - "--help_create_dataset", - "--help_create_index", - "--help_extract", - "--help_preprocess", - "--help_separate_music", - "--help_train", - "--help_create_reference", - "--help" -] - -COMMANDS = { - "--audio_effects": "main.inference.audio_effects", - "--convert": "main.inference.conversion.convert", - "--create_dataset": "main.inference.create_dataset", - "--create_index": "main.inference.create_index", - "--extract": "main.inference.extracting.extract", - "--preprocess": "main.inference.preprocess.preprocess", - "--separate_music": "main.inference.separate_music", - "--train": "main.inference.training.train", - "--create_reference": "main.inference.create_reference", -} - -if argv is None or argv not in list(COMMANDS) and "--help" not in argv: - print("Cú pháp không hợp lệ! Sử dụng --help để biết thêm") - sys.exit() - -if argv_help_is_allows[0] == argv: - print("""Các tham số của `--audio_effects`: - 1. Đường dẫn tệp: - - `--input_path` (bắt buộc): Đường dẫn đến tệp âm thanh đầu vào. - - `--output_path` (mặc định: `./audios/apply_effects.wav`): Đường dẫn lưu tệp đầu ra. - - `--export_format` (mặc định: `wav`): Định dạng xuất tệp (`wav`, `mp3`, ...). - - 2. Lấy mẫu lại: - - `--resample` (mặc định: `False`): Có lấy mẫu lại hay không. - - `--resample_sr` (mặc định: `0`): Tần số lấy mẫu mới (Hz). - - 3. Chorus: - - `--chorus`: Bật/tắt chorus. - - `--chorus_depth`, `--chorus_rate`, `--chorus_mix`, `--chorus_delay`, `--chorus_feedback`: Các thông số điều chỉnh chorus. - - 4. Distortion: - - `--distortion`: Bật/tắt distortion. - - `--drive_db`: Mức độ méo âm thanh. - - 5. Reverb: - - `--reverb`: Bật/tắt hồi âm. - - `--reverb_room_size`, `--reverb_damping`, `--reverb_wet_level`, `--reverb_dry_level`, `--reverb_width`, `--reverb_freeze_mode`: Điều chỉnh hồi âm. - - 6. Pitchshift: - - `--pitchshift`: Bật/tắt thay đổi cao độ. - - `--pitch_shift`: Giá trị dịch cao độ. - - 7. Delay: - - `--delay`: Bật/tắt delay. - - `--delay_seconds`, `--delay_feedback`, `--delay_mix`: Điều chỉnh thời gian trễ, phản hồi và hòa trộn. - - 8. Compressor: - - `--compressor`: Bật/tắt compressor. - - `--compressor_threshold`, `--compressor_ratio`, `--compressor_attack_ms`, `--compressor_release_ms`: Các thông số nén. - - 9. Limiter: - - `--limiter`: Bật/tắt giới hạn mức âm thanh. - - `--limiter_threshold`, `--limiter_release`: Ngưỡng giới hạn và thời gian nhả. - - 10. Gain (Khuếch đại): - - `--gain`: Bật/tắt gain. - - `--gain_db`: Mức gain (dB). - - 11. Bitcrush: - - `--bitcrush`: Bật/tắt hiệu ứng giảm độ phân giải. - - `--bitcrush_bit_depth`: Số bit của bitcrush. - - 12. Clipping: - - `--clipping`: Bật/tắt cắt âm thanh. - - `--clipping_threshold`: Ngưỡng clipping. - - 13. Phaser: - - `--phaser`: Bật/tắt hiệu ứng phaser. - - `--phaser_rate_hz`, `--phaser_depth`, `--phaser_centre_frequency_hz`, `--phaser_feedback`, `--phaser_mix`: Điều chỉnh hiệu ứng phaser. - - 14. Boost bass & treble: - - `--treble_bass_boost`: Bật/tắt tăng cường âm bass và treble. - - `--bass_boost_db`, `--bass_boost_frequency`, `--treble_boost_db`, `--treble_boost_frequency`: Các thông số tăng bass và treble. - - 15. Fade in & fade out: - - `--fade_in_out`: Bật/tắt hiệu ứng fade. - - `--fade_in_duration`, `--fade_out_duration`: Thời gian fade vào/ra. - - 16. Kết hợp âm thanh: - - `--audio_combination`: Bật/tắt ghép nhiều tệp âm thanh. - - `--audio_combination_input`: Đường dẫn tệp âm thanh bổ sung. - - `--main_volume`: Âm lượng của âm thanh chính. - - `--combination_volume`:: Âm lượng của âm thanh cần kết hợp. - """) - sys.exit() -elif argv_help_is_allows[1] == argv: - print("""Các tham số của --convert: - 1. Cấu hình xử lí giọng nói: - - `--pitch` (mặc định: `0`): Điều chỉnh cao độ. - - `--filter_radius` (mặc định: `3`): Độ mượt của đường F0. - - `--index_rate` (mặc định: `0.5`): Tỷ lệ sử dụng chỉ mục giọng nói. - - `--rms_mix_rate` (mặc định: `1`): Hệ số điều chỉnh biên độ âm lượng. - - `--protect` (mặc định: `0.33`): Bảo vệ phụ âm. - - `--hop_length` (mặc định: `64`): Bước nhảy khi xử lí âm thanh. - - `--sid` (mặc định: `0`): Mã số nhận dạng giọng nói đối với mô hình đa giọng. - - `--noise_scale` (mặc định: `0.35`): Tùy chỉnh đặc biệt cho mô hình SVC gây ảnh hưởng trực tiếp đến chất lượng của âm thanh, bạn không nên điều chỉnh nó. - - 2. Cấu hình F0: - - `--f0_method` (mặc định: `rmvpe`): Phương pháp dự đoán F0 (`pm-ac`, `pm-cc`, `pm-shs`, `dio`, `dio-stonemask`, `mangio-crepe-tiny`, `mangio-crepe-small`, `mangio-crepe-medium`, `mangio-crepe-large`, `mangio-crepe-full`, `crepe-tiny`, `crepe-small`, `crepe-medium`, `crepe-large`, `crepe-full`, `fcpe`, `fcpe-legacy`, `fcpe-previous`, `rmvpe`, `rmvpe-medfilt`, `rmvpe-mix`, `rmvpe-mix-medfilt`, `hpa-rmvpe`, `hpa-rmvpe-medfilt`, `hpa-rmvpe-previous`, `hpa-rmvpe-previous-medfilt`, `harvest`, `harvest-stonemask`, `yin`, `pyin`, `swipe`, `swipe-stonemask`, `piptrack`, `penn`, `mangio-penn`, `djcm`, `djcm-medfilt`, `djcm-svs`, `djcm-svs-medfilt`, `swift`, `pesto`). - - `--f0_autotune` (mặc định: `False`): Có tự động điều chỉnh F0 hay không. - - `--f0_autotune_strength` (mặc định: `1`): Cường độ hiệu chỉnh tự động F0. - - `--f0_file` (mặc định: ``): Đường dẫn tệp F0 có sẵn. - - `--predictor_onnx` (mặc định: `False`): Có sử dụng chế độ tăng tốc bằng ONNX của trích xuất cao độ hay không. - - `--proposal_pitch` (mặc định: `False`): Đề xuất cao độ thay vì điều chỉnh thủ công. - - `--proposal_pitch_threshold` (mặc định: `0.0`): Ngưỡng tần số ước tính cao độ. - - `--alpha` (mặc định: `0.5`): Ngưỡng trộn cao độ khi ước tính cao độ hybrid. - - 3. Mô hình nhúng: - - `--embedder_model` (mặc định: `hubert_base`): Mô hình nhúng sử dụng. - - `--embedders_mode` (mặc định: `fairseq`): Chế độ nhúng (`fairseq`, `transformers`, `onnx`, `whisper`). - - `--embedders_mix` (mặc định: `False`): Hòa trộn tầng. - - `--embedders_mix_layers` (mặc định: `9`): Tầng hòa trộn. - - `--embedders_mix_ratio` (mặc định: `0.5`): Tỉ lệ hòa trộn. - - 4. Đường dẫn tệp: - - `--input_path` (bắt buộc): Đường dẫn tệp âm thanh đầu vào. - - `--output_path` (mặc định: `./audios/output.wav`): Đường dẫn lưu tệp đầu ra. - - `--export_format` (mặc định: `wav`): Định dạng xuất tệp. - - `--pth_path` (bắt buộc): Đường dẫn đến tệp mô hình `.pth`. - - `--index_path` (mặc định: `None`): Đường dẫn tệp chỉ mục (nếu có). - - 5. Làm sạch âm thanh: - - `--clean_audio` (mặc định: `False`): Có áp dụng làm sạch âm thanh không. - - `--clean_strength` (mặc định: `0.7`): Mức độ làm sạch. - - 6. Resampling & chia nhỏ âm thanh: - - `--resample_sr` (mặc định: `0`): Tần số lấy mẫu mới (0 nghĩa là giữ nguyên). - - `--split_audio` (mặc định: `False`): Có chia nhỏ audio trước khi xử lí không. - - 7. Kiểm tra & tối ưu hóa: - - `--checkpointing` (mặc định: `False`): Bật/tắt checkpointing để tiết kiệm RAM. - - 8. Dịch formant: - - `--formant_shifting` (mặc định: `False`): Có bật hiệu ứng dịch formant không. - - `--formant_qfrency` (mặc định: `0.8`): Hệ số dịch formant theo tần số. - - `--formant_timbre` (mặc định: `0.8`): Hệ số thay đổi màu sắc giọng. - """) - sys.exit() -elif argv_help_is_allows[2] == argv: - print("""Các tham số của --create_dataset: - 1. Đường dẫn & cấu hình dataset: - - `--input_data` (bắt buộc): Đường dẫn liên kết đến âm thanh (Liên kết Youtube, có thể dùng dấu `,` để dùng nhiều liên kết). - - `--output_dirs` (mặc định: `./dataset`): Thư mục xuất dữ liệu đầu ra. - - `--sample_rate` (mặc định: `48000`): Tần số lấy mẫu cho âm thanh. - - 2. Làm sạch dữ liệu: - - `--clean_dataset` (mặc định: `False`): Có áp dụng làm sạch dữ liệu hay không. - - `--clean_strength` (mặc định: `0.7`): Mức độ làm sạch dữ liệu. - - 3. Tách giọng & hiệu ứng: - - `--separate` (mặc định: `True`): có tách nhạc hay không. - - `--separator_reverb` (mặc định: `False`): Có tách vang giọng không. - - `--model_name` (mặc định: `MDXNET_Main`): Mô hình tách nhạc ('Main_340', 'Main_390', 'Main_406', 'Main_427', 'Main_438', 'Inst_full_292', 'Inst_HQ_1', 'Inst_HQ_2', 'Inst_HQ_3', 'Inst_HQ_4', 'Inst_HQ_5', 'Kim_Vocal_1', 'Kim_Vocal_2', 'Kim_Inst', 'Inst_187_beta', 'Inst_82_beta', 'Inst_90_beta', 'Voc_FT', 'Crowd_HQ', 'MDXNET_9482', 'Inst_1', 'Inst_2', 'Inst_3', 'MDXNET_1_9703', 'MDXNET_2_9682', 'MDXNET_3_9662', 'Inst_Main', 'MDXNET_Main', 'HT-Tuned', 'HT-Normal', 'HD_MMI', 'HT_6S', 'HP-1', 'HP-2', 'HP-Vocal-1', 'HP-Vocal-2', 'HP2-1', 'HP2-2', 'HP2-3', 'SP-2B-1', 'SP-2B-2', 'SP-3B-1', 'SP-4B-1', 'SP-4B-2', 'SP-MID-1', 'SP-MID-2'). - - `--reverb_model` (mặc định: `MDX-Reverb`): Mô hình tách nhạc ("MDX-Reverb", 'VR-Reverb', 'Echo-Aggressive', 'Echo-Normal'). - - `--denoise_model` (mặc định: `Normal`): Mô hình tách nhạc ('Lite', 'Normal'). - - 4. Cấu hình xử lí âm thanh: - - `--shifts` (mặc định: `2`): Số lượng dự đoán. - - `--batch_size` (mặc định: `1`): Kích thước lô. - - `--overlap` (mặc định: `0.25`): Mức độ chồng lấn giữa các đoạn. - - `--aggression` (mặc định: `5`): Cường độ chiết xuất thân chính. - - `--hop_length` (mặc định: `1024`): Bước nhảy MDX khi xử lí. - - `--window_size` (mặc định: `512`): Kích thước cửa sổ. - - `--segments_size` (mặc định: `256`): Kích thước phân đoạn âm thanh. - - `--post_process_threshold` (mặc định: `0.2`): Mức độ xử lí hậu kỳ sau khi tách nhạc. - - 5. Cấu hình xử lí âm thanh khác: - - `--enable_tta` (mặc định: `False`): Tăng cường suy luận. - - `--enable_denoise` (mặc định: `False`): Khữ tách nhạc. - - `--high_end_process` (mặc định: `False`): Xử lí dải cao. - - `--enable_post_process` (mặc định: `False`): Hậu xử lí. - - 6. Bỏ qua phần âm thanh: - - `--skip_seconds` (mặc định: `False`): Có bỏ qua giây âm thanh nào không. - - `--skip_start_audios` (mặc định: `0`): Thời gian (giây) cần bỏ qua ở đầu audio. - - `--skip_end_audios` (mặc định: `0`): Thời gian (giây) cần bỏ qua ở cuối audio. - """) - sys.exit() -elif argv_help_is_allows[3] == argv: - print("""Các tham số của --create_index: - 1. Thông tin mô hình: - - `--model_name` (bắt buộc): Tên mô hình. - - `--rvc_version` (mặc định: `v2`): Phiên bản (`v1`, `v2`). - - `--index_algorithm` (mặc định: `Auto`): Thuật toán index sử dụng (`Auto`, `Faiss`, `KMeans`). - """) - sys.exit() -elif argv_help_is_allows[4] == argv: - print("""Các tham số của --extract: - 1. Thông tin mô hình: - - `--model_name` (bắt buộc): Tên mô hình. - - `--rvc_version` (mặc định: `v2`): Phiên bản mô hình (`v1`, `v2`). - - 2. Cấu hình F0: - - `--f0_method` (mặc định: `rmvpe`): Phương pháp dự đoán F0 (`pm-ac`, `pm-cc`, `pm-shs`, `dio`, `dio-stonemask`, `mangio-crepe-tiny`, `mangio-crepe-small`, `mangio-crepe-medium`, `mangio-crepe-large`, `mangio-crepe-full`, `crepe-tiny`, `crepe-small`, `crepe-medium`, `crepe-large`, `crepe-full`, `fcpe`, `fcpe-legacy`, `fcpe-previous`, `rmvpe`, `rmvpe-medfilt`, `rmvpe-mix`, `rmvpe-mix-medfilt`, `hpa-rmvpe`, `hpa-rmvpe-medfilt`, `hpa-rmvpe-previous`, `hpa-rmvpe-previous-medfilt`, `harvest`, `harvest-stonemask`, `yin`, `pyin`, `swipe`, `swipe-stonemask`, `piptrack`, `penn`, `mangio-penn`, `djcm`, `djcm-medfilt`, `djcm-svs`, `djcm-svs-medfilt`, `swift`, `pesto`). - - `--predictor_onnx` (mặc định: `False`): Có sử dụng chế độ tăng tốc bằng ONNX của trích xuất cao độ hay không. - - `--pitch_guidance` (mặc định: `True`): Có sử dụng hướng dẫn cao độ hay không. - - `--f0_autotune` (mặc định: `False`): Có tự động điều chỉnh F0 hay không. - - `--f0_autotune_strength` (mặc định: `1`): Cường độ hiệu chỉnh tự động F0. - - `--alpha` (mặc định: `0.5`): Ngưỡng trộn cao độ khi ước tính cao độ hybrid. - - 3. Cấu hình xử lí: - - `--hop_length` (mặc định: `128`): Độ dài bước nhảy trong quá trình xử lí. - - `--cpu_cores` (mặc định: `2`): Số lượng luồng CPU sử dụng. - - `--gpu` (mặc định: `-`): Chỉ định GPU sử dụng (ví dụ: `0` cho GPU đầu tiên, `-` để tắt GPU). - - `--sample_rate` (bắt buộc): Tần số lấy mẫu của âm thanh đầu vào. - - `--include_mutes` (mặc định: `2`): Số lượng tệp im lặng được sử dụng huấn luyện. - - 4. Cấu hình nhúng: - - `--embedder_model` (mặc định: `hubert_base`): Tên mô hình nhúng. - - `--embedders_mode` (mặc định: `fairseq`): Chế độ nhúng (`fairseq`, `transformers`, `onnx`, `whisper`). - - `--embedders_mix` (mặc định: `False`): Hòa trộn tầng. - - `--embedders_mix_layers` (mặc định: `9`): Tầng hòa trộn. - - `--embedders_mix_ratio` (mặc định: `0.5`): Tỉ lệ hòa trộn. - - 4. RMS: - - `--rms_extract` (mặc định: False): Trích xuất thêm năng lượng rms. - """) - sys.exit() -elif argv_help_is_allows[5] == argv: - print("""Các tham số của --preprocess: - 1. Thông tin mô hình: - - `--model_name` (bắt buộc): Tên mô hình. - - 2. Cấu hình dữ liệu: - - `--dataset_path` (mặc định: `./dataset`): Đường dẫn thư mục chứa tệp dữ liệu. - - `--sample_rate` (bắt buộc): Tần số lấy mẫu của dữ liệu âm thanh. - - 3. Cấu hình xử lí: - - `--cpu_cores` (mặc định: `2`): Số lượng luồng CPU sử dụng. - - `--cut_preprocess` (mặc định: `Automatic`): Cách cắt dữ liệu tiền xử lí (`Automatic`, `Simple`, `Skip`). - - `--process_effects` (mặc định: `False`): Có áp dụng tiền xử lí hay không. - - `--clean_dataset` (mặc định: `False`): Có làm sạch tệp dữ liệu hay không. - - `--clean_strength` (mặc định: `0.7`): Độ mạnh của quá trình làm sạch dữ liệu. - - 4. Cấu hình khác: - - `--chunk_len` (mặc định: `3.0`): Độ dài của đoạn âm thanh cho phương pháp 'Simple'. - - `--overlap_len` (mặc định: `0.3`): Độ dài của phần chồng chéo giữa các lát cắt đối với phương pháp 'Simple'. - - `--normalization_mode` (mặc định: `none`): Có xử lí chuẩn hóa âm thanh không (`none`, `pre`, `post`) - """) - sys.exit() -elif argv_help_is_allows[6] == argv: - print("""Các tham số của --separate_music: - 1. Cấu hình đầu vào, đầu ra: - - `--input_path` (bắt buộc): Đường dẫn tệp âm thanh đầu vào. - - `--output_dirs` (mặc định: `./audios`): Thư mục lưu tệp đầu ra. - - `--export_format` (mặc định: `wav`): Định dạng xuất tệp (`wav`, `mp3`,...). - - `--sample_rate` (mặc định: `44100`): Tần số lấy mẫu của âm thanh đầu ra. - - 2. Cấu hình mô hình: - - `--model_name` (mặc định: `MDXNET_Main`): Mô hình tách nhạc ('Main_340', 'Main_390', 'Main_406', 'Main_427', 'Main_438', 'Inst_full_292', 'Inst_HQ_1', 'Inst_HQ_2', 'Inst_HQ_3', 'Inst_HQ_4', 'Inst_HQ_5', 'Kim_Vocal_1', 'Kim_Vocal_2', 'Kim_Inst', 'Inst_187_beta', 'Inst_82_beta', 'Inst_90_beta', 'Voc_FT', 'Crowd_HQ', 'MDXNET_9482', 'Inst_1', 'Inst_2', 'Inst_3', 'MDXNET_1_9703', 'MDXNET_2_9682', 'MDXNET_3_9662', 'Inst_Main', 'MDXNET_Main', 'HT-Tuned', 'HT-Normal', 'HD_MMI', 'HT_6S', 'HP-1', 'HP-2', 'HP-Vocal-1', 'HP-Vocal-2', 'HP2-1', 'HP2-2', 'HP2-3', 'SP-2B-1', 'SP-2B-2', 'SP-3B-1', 'SP-4B-1', 'SP-4B-2', 'SP-MID-1', 'SP-MID-2'). - - `--karaoke_model` (mặc định: `MDX-Version-1`): Mô hình tách nhạc ('MDX-Version-1', 'MDX-Version-2', 'VR-Version-1', 'VR-Version-2'). - - `--reverb_model` (mặc định: `MDX-Reverb`): Mô hình tách nhạc ("MDX-Reverb", 'VR-Reverb', 'Echo-Aggressive', 'Echo-Normal'). - - `--denoise_model` (mặc định: `Normal`): Mô hình tách nhạc ('Lite', 'Normal'). - - 3. Cấu hình xử lí âm thanh: - - `--shifts` (mặc định: `2`): Số lượng dự đoán. - - `--batch_size` (mặc định: `1`): Kích thước lô. - - `--overlap` (mặc định: `0.25`): Mức độ chồng lấn giữa các đoạn. - - `--aggression` (mặc định: `5`): Cường độ chiết xuất thân chính. - - `--hop_length` (mặc định: `1024`): Bước nhảy MDX khi xử lí. - - `--window_size` (mặc định: `512`): Kích thước cửa sổ. - - `--segments_size` (mặc định: `256`): Kích thước phân đoạn âm thanh. - - `--post_process_threshold` (mặc định: `0.2`): Mức độ xử lí hậu kỳ sau khi tách nhạc. - - 4. Cấu hình xử lí âm thanh khác: - - `--enable_tta` (mặc định: `False`): Tăng cường suy luận. - - `--enable_denoise` (mặc định: `False`): Khữ tách nhạc. - - `--high_end_process` (mặc định: `False`): Xử lí dải cao. - - `--enable_post_process` (mặc định: `False`): Hậu xử lí. - - `--separate_backing` (mặc định: `False`): Tách bè giọng. - - `--separate_reverb` (mặc định: `False`): Tách vang giọng. - """) - sys.exit() -elif argv_help_is_allows[7] == argv: - print("""Các tham số của --train: - 1. Cấu hình mô hình: - - `--model_name` (bắt buộc): Tên mô hình. - - `--rvc_version` (mặc định: `v2`): Phiên bản mô hình (`v1`, `v2`). - - `--model_author` (tùy chọn): Tác giả của mô hình. - - 2. Cấu hình lưu: - - `--save_every_epoch` (bắt buộc): Số kỷ nguyên giữa mỗi lần lưu. - - `--save_only_latest` (mặc định: `True`): Chỉ lưu điểm mới nhất. - - `--save_every_weights` (mặc định: `True`): Lưu tất cả trọng số của mô hình. - - 3. Cấu hình huấn luyện: - - `--total_epoch` (mặc định: `300`): Tổng số kỷ nguyên huấn luyện. - - `--batch_size` (mặc định: `8`): Kích thước lô trong quá trình huấn luyện. - - 4. Cấu hình thiết bị: - - `--gpu` (mặc định: `0`): Chỉ định GPU để sử dụng (số hoặc `-` nếu không dùng GPU). - - `--cache_data_in_gpu` (mặc định: `False`): Lưu dữ liệu vào GPU để tăng tốc. - - 5. Cấu hình huấn luyện nâng cao: - - `--pitch_guidance` (mặc định: `True`): Sử dụng hướng dẫn cao độ. - - `--g_pretrained_path` (mặc định: ``): Đường dẫn đến trọng số G đã huấn luyện trước. - - `--d_pretrained_path` (mặc định: ``): Đường dẫn đến trọng số D đã huấn luyện trước. - - `--vocoder` (mặc định: `Default`): Bộ mã hóa được sử dụng (`Default`, `MRF-HiFi-GAN`, `RefineGAN`). - - `--energy_use` (mặc định: `False`): Sử dụng năng lượng rms. - - `--architecture` (mặc định: `RVC`): Kiến trúc của mô hình giọng nói (`RVC`, `SVC`). - - 6. Phát hiện huấn luyện quá mức: - - `--overtraining_detector` (mặc định: `False`): Bật/tắt chế độ phát hiện huấn luyện quá mức. - - `--overtraining_threshold` (mặc định: `50`): Ngưỡng để xác định huấn luyện quá mức. - - 7. Xử lí dữ liệu: - - `--cleanup` (mặc định: `False`): Dọn dẹp tệp huấn luyện cũ để tiến hành huấn luyện lại từ đầu. - - 8. Tối ưu: - - `--checkpointing` (mặc định: `False`): Bật/tắt checkpointing để tiết kiệm RAM. - - `--deterministic` (mặc định: `False`): Khi bật sẽ sử dụng các thuật toán có tính xác định cao, đảm bảo rằng mỗi lần chạy cùng một dữ liệu đầu vào sẽ cho kết quả giống nhau. - - `--benchmark` (mặc định: `False`): Khi bật sẽ thử nghiệm và chọn thuật toán tối ưu nhất cho phần cứng và kích thước cụ thể. - - `--optimizer` (mặc định: `AdamW`): Trình tối ưu hóa được sử dụng (`AdamW`, `RAdam`, `AnyPrecisionAdamW`, `AdaBelief`, `AdaBeliefV2`). - - `--multiscale_mel_loss` (mặc định: `False`): So sánh phổ Mel của âm thanh thật và âm thanh giả ở nhiều thang độ khác nhau. Giúp mô hình học được chi tiết âm sắc, độ sáng và cấu trúc tần số tốt hơn, từ đó cải thiện chất lượng và độ tự nhiên của giọng nói đầu ra. - - `--use_cosine_annealing_lr` (mặc định: `False`): Sử dụng phương pháp giảm LR bằng Cosine Annealing có thể giúp cải thiện chất lượng phát âm. - - 9. Bộ tham chiếu: - - `--use_custom_reference` (mặc định: `False`): Có tùy chỉnh bộ tham chiếu hay không. - - `--reference_path` (mặc định: `False`): Đường dẫn đến bộ tham chiếu. - """) - sys.exit() -elif argv_help_is_allows[8] == argv: - print("""Các tham số của --create_reference: - 1. Đường dẫn tệp: - - `--audio_path` (bắt buộc): Đường dẫn tệp âm thanh đầu vào. - - `--reference_name` (mặc định: `reference`): Đường dẫn lưu bộ tham chiếu đầu ra. - - 2. Cấu hình bộ tham chiếu: - - `--pitch_guidance` (mặc định: `True`): Sử dụng hướng dẫn cao độ. - - `--energy_use` (mặc định: `False`): Sử dụng năng lượng rms. - - `--version` (mặc định: `v2`): Phiên bản mô hình (`v1`, `v2`). - - 3. Cấu hình nhúng: - - `--embedder_model` (mặc định: `hubert_base`): Tên mô hình nhúng. - - `--embedders_mode` (mặc định: `fairseq`): Chế độ nhúng (`fairseq`, `transformers`, `onnx`, `whisper`). - - `--embedders_mix` (mặc định: `False`): Hòa trộn tầng. - - `--embedders_mix_layers` (mặc định: `9`): Tầng hòa trộn. - - `--embedders_mix_ratio` (mặc định: `0.5`): Tỉ lệ hòa trộn. - - 4. Cấu hình F0: - - `--f0_method` (mặc định: `rmvpe`): Phương pháp dự đoán F0 (`pm-ac`, `pm-cc`, `pm-shs`, `dio`, `dio-stonemask`, `mangio-crepe-tiny`, `mangio-crepe-small`, `mangio-crepe-medium`, `mangio-crepe-large`, `mangio-crepe-full`, `crepe-tiny`, `crepe-small`, `crepe-medium`, `crepe-large`, `crepe-full`, `fcpe`, `fcpe-legacy`, `fcpe-previous`, `rmvpe`, `rmvpe-medfilt`, `rmvpe-mix`, `rmvpe-mix-medfilt`, `hpa-rmvpe`, `hpa-rmvpe-medfilt`, `hpa-rmvpe-previous`, `hpa-rmvpe-previous-medfilt`, `harvest`, `harvest-stonemask`, `yin`, `pyin`, `swipe`, `swipe-stonemask`, `piptrack`, `penn`, `mangio-penn`, `djcm`, `djcm-medfilt`, `djcm-svs`, `djcm-svs-medfilt`, `swift`, `pesto`). - - `--predictor_onnx` (mặc định: `False`): Có sử dụng chế độ tăng tốc bằng ONNX của trích xuất cao độ hay không. - - `--f0_up_key` (mặc định: `0`): Điều chỉnh cao độ. - - `--filter_radius` (mặc định: `3`): Độ mượt của đường F0. - - `--f0_autotune` (mặc định: `False`): Có tự động điều chỉnh F0 hay không. - - `--f0_autotune_strength` (mặc định: `1`): Cường độ hiệu chỉnh tự động F0. - - `--f0_file` (mặc định: ``): Đường dẫn tệp F0 có sẵn. - - `--proposal_pitch` (mặc định: `False`): Đề xuất cao độ thay vì điều chỉnh thủ công. - - `--proposal_pitch_threshold` (mặc định: `0.0`): Ngưỡng tần số ước tính cao độ. - - `--alpha` (mặc định: `0.5`): Ngưỡng trộn cao độ khi ước tính cao độ hybrid. - """) - sys.exit() -elif argv_help_is_allows[9] == argv: - print("""Sử dụng: - 1. `--help_audio_effects`: Trợ giúp về phần thêm hiệu ứng âm thanh. - 2. `--help_convert`: Trợ giúp về chuyển đổi âm thanh. - 3. `--help_create_dataset`: Trợ giúp về tạo dữ liệu huấn luyện. - 4. `--help_create_index`: Trợ giúp về tạo chỉ mục. - 5. `--help_extract`: Trợ giúp về trích xuất dữ liệu huấn luyện. - 6. `--help_preprocess`: Trợ giúp về xử lí trước dữ liệu. - 7. `--help_separate_music`: Trợ giúp về tách nhạc. - 8. `--help_train`: Trợ giúp về huấn luyện mô hình. - 9. `--help_create_reference`: Trợ giúp về tạo bộ tham chiếu. - """) - sys.exit() -elif argv in COMMANDS: - import importlib - - module = importlib.import_module(COMMANDS[argv]) - main = module.main -else: - main = lambda: print("Cú pháp không hợp lệ!"); sys.exit() - -if __name__ == "__main__": - import torch.multiprocessing as mp - - try: - if "--train" in argv: mp.set_start_method("spawn") - if "--preprocess" in argv or "--extract" in argv: mp.set_start_method("spawn", force=True) - except: - pass - - main() \ No newline at end of file diff --git a/main/app/run_tensorboard.py b/main/app/run_tensorboard.py deleted file mode 100644 index 1cc07da8d6002b535d7c9c1b600d80aeb798fd4f..0000000000000000000000000000000000000000 --- a/main/app/run_tensorboard.py +++ /dev/null @@ -1,39 +0,0 @@ -import os -import sys -import time -import logging -import warnings -import webbrowser - -from tensorboard import program - -sys.path.append(os.getcwd()) - -from main.app.variables import config, translations, logger - -def launch_tensorboard(): - if not config.debug_mode: - warnings.filterwarnings("ignore") - for l in ["root", "tensorboard"]: - logging.getLogger(l).setLevel(logging.ERROR) - - tb = program.TensorBoard() - tb.configure( - argv=[ - None, - "--logdir", config.configs["logs_path"], - f"--port={config.configs['tensorboard_port']}" - ] - ) - url = tb.launch() - - logger.info(f"{translations['tensorboard_url']}: {url}") - if "--open" in sys.argv: webbrowser.open(url) - - return url - -if __name__ == "__main__": - launch_tensorboard() - - while 1: - time.sleep(5) \ No newline at end of file diff --git a/main/app/tabs/downloads/downloads.py b/main/app/tabs/downloads/downloads.py deleted file mode 100644 index 4e6db4b3357f894901e3c9a37086e6997c2f1efd..0000000000000000000000000000000000000000 --- a/main/app/tabs/downloads/downloads.py +++ /dev/null @@ -1,330 +0,0 @@ -import os -import sys - -import gradio as gr - -sys.path.append(os.getcwd()) - -from main.app.variables import ( - models, - configs, - translations, - model_options -) - -from main.app.core.downloads import ( - search_models, - download_model, - download_pretrained_model -) - -from main.app.core.ui import ( - shutil_move, - change_download_choices, - change_download_pretrained_choices -) - -from main.app.core.process import ( - save_drop_model, - fetch_pretrained_data, - update_sample_rate_dropdown -) - -def download_tab(): - with gr.TabItem( - translations["downloads"] - ): - gr.Markdown(translations["download_markdown"]) - with gr.Row(): - gr.Markdown(translations["download_markdown_2"]) - with gr.Row(): - with gr.Accordion( - translations["model_download"], - open=True - ): - with gr.Row(): - download_method = gr.Radio( - label=translations["model_download_select"], - choices=[ - translations["download_url"], - translations["download_from_csv"], - translations["search_models"], - translations["upload"] - ], - interactive=True, - value=translations["download_url"] - ) - with gr.Row(): - gr.Markdown("___") - with gr.Column(): - with gr.Row(): - download_model_url = gr.Textbox( - label=translations["model_url"], - value="", - placeholder="https://...", - scale=6 - ) - download_model_name = gr.Textbox( - label=translations["modelname"], - value="", - placeholder=translations["modelname"], - scale=2 - ) - download_url_button = gr.Button( - value=translations["downloads"], - scale=2 - ) - with gr.Column(): - download_csv_model = gr.Dropdown( - choices=models.keys(), - label=translations["model_warehouse"], - scale=8, - allow_custom_value=True, - visible=False - ) - download_csv_button = gr.Button( - value=translations["get_model"], - scale=2, - variant="primary", - visible=False - ) - with gr.Column(): - search_model_name = gr.Textbox( - label=translations["name_to_search"], - placeholder=translations["modelname"], - interactive=True, - scale=8, - visible=False - ) - search_button = gr.Button( - translations["search_2"], - scale=2, - visible=False - ) - search_dropdown = gr.Dropdown( - label=translations["select_download_model"], - value="", - choices=[], - allow_custom_value=True, - interactive=False, - visible=False - ) - search_download_model = gr.Button( - translations["downloads"], - variant="primary", - visible=False - ) - with gr.Column(): - upload_model_files = gr.Files( - label=translations["drop_model"], - file_types=[".pth", ".onnx", ".index", ".zip"], - visible=False - ) - with gr.Row(): - with gr.Accordion( - translations["download_pretrained_2"], - open=True - ): - with gr.Row(): - download_pretrain_method = gr.Radio( - label=translations["model_download_select"], - choices=[ - translations["download_url"], - translations["list_model"], - translations["upload"] - ], - value=translations["download_url"], - interactive=True - ) - with gr.Row(): - gr.Markdown("___") - with gr.Column(): - with gr.Row(): - download_pretrain_disc_url = gr.Textbox( - label=translations["pretrained_url"].format(dg="D"), - value="", - placeholder="https://...", - interactive=True, - scale=4 - ) - download_pretrain_gen_url = gr.Textbox( - label=translations["pretrained_url"].format(dg="G"), - value="", - placeholder="https://...", - interactive=True, - scale=4 - ) - download_pretrain_button = gr.Button( - translations["downloads"], - scale=2, - visible=True - ) - with gr.Column(): - with gr.Row(): - download_pretrain_choices = gr.Dropdown( - label=translations["select_pretrain"], - info=translations["select_pretrain_info"], - choices=list(fetch_pretrained_data().keys()), - value="Titan_Medium", - allow_custom_value=True, - interactive=True, - scale=6, - visible=False - ) - download_pretrain_sample_rate = gr.Dropdown( - label=translations["pretrain_sr"], - info=translations["pretrain_sr"], - choices=["48k", "40k", "32k"], - value="48k", - interactive=True, - visible=False - ) - download_pretrain_choices_button = gr.Button( - translations["downloads"], - scale=2, - variant="primary", - visible=False - ) - with gr.Row(): - upload_pretrains = gr.Files( - label=translations["drop_pretrain"].format(dg="G, D"), - file_types=[".pth"], - visible=False - ) - with gr.Row(): - download_url_button.click( - fn=download_model, - inputs=[ - download_model_url, - download_model_name - ], - outputs=[download_model_url], - api_name="download_model" - ) - download_csv_button.click( - fn=lambda model: download_model( - models[model], - model - ), - inputs=[ - download_csv_model - ], - outputs=[ - download_csv_model - ], - api_name="download_browser" - ) - with gr.Row(): - download_method.change( - fn=change_download_choices, - inputs=[ - download_method - ], - outputs=[ - download_model_url, - download_model_name, - download_url_button, - download_csv_model, - download_csv_button, - search_model_name, - search_button, - search_dropdown, - search_download_model, - upload_model_files - ] - ) - search_button.click( - fn=search_models, - inputs=[ - search_model_name - ], - outputs=[ - search_dropdown, - search_download_model - ] - ) - with gr.Row(): - upload_model_files.upload( - fn=save_drop_model, - inputs=[ - upload_model_files - ], - outputs=[ - upload_model_files - ] - ) - search_download_model.click( - fn=lambda model: download_model( - model_options[model], - model - ), - inputs=[ - search_dropdown - ], - outputs=[ - search_dropdown - ], - api_name="search_models" - ) - with gr.Row(): - download_pretrain_method.change( - fn=change_download_pretrained_choices, - inputs=[ - download_pretrain_method - ], - outputs=[ - download_pretrain_disc_url, - download_pretrain_gen_url, - download_pretrain_button, - download_pretrain_choices, - download_pretrain_sample_rate, - download_pretrain_choices_button, - upload_pretrains - ] - ) - download_pretrain_choices.change( - fn=update_sample_rate_dropdown, - inputs=[ - download_pretrain_choices - ], - outputs=[ - download_pretrain_sample_rate - ] - ) - with gr.Row(): - download_pretrain_button.click( - fn=download_pretrained_model, - inputs=[ - download_pretrain_method, - download_pretrain_disc_url, - download_pretrain_gen_url - ], - outputs=[ - download_pretrain_disc_url, - download_pretrain_gen_url - ], - api_name="download_pretrain_link" - ) - download_pretrain_choices_button.click( - fn=download_pretrained_model, - inputs=[ - download_pretrain_method, - download_pretrain_choices, - download_pretrain_sample_rate - ], - outputs=[ - download_pretrain_choices - ], - api_name="download_pretrain_choices" - ) - upload_pretrains.upload( - fn=lambda upload_pretrains: [ - shutil_move(pretrain.name, configs["pretrained_custom_path"]) - for pretrain in upload_pretrains - ], - inputs=[ - upload_pretrains - ], - outputs=[], - api_name="upload_pretrain" - ) \ No newline at end of file diff --git a/main/app/tabs/editing/child/audio_effects.py b/main/app/tabs/editing/child/audio_effects.py deleted file mode 100644 index 050ca6947c5f674dbc6cd6dfdcc9df3b4237eb1d..0000000000000000000000000000000000000000 --- a/main/app/tabs/editing/child/audio_effects.py +++ /dev/null @@ -1,1165 +0,0 @@ -import os -import sys - -import gradio as gr - -sys.path.append(os.getcwd()) - -from main.app.core.editing import audio_effects - -from main.app.core.presets import ( - audio_effect_load_presets, - audio_effect_save_presets -) - -from main.app.core.ui import ( - visible, - shutil_move, - change_audios_choices, - change_effect_preset_choices -) - -from main.app.variables import ( - configs, - file_types, - translations, - audio_params, - paths_for_files, - sample_rate_choice, - export_format_choices, - audio_effect_presets_file -) - -def audio_effects_tab(): - with gr.Row(): - gr.Markdown(translations["audio_effects_edit"]) - with gr.Row(): - with gr.Column(): - with gr.Row(): - reverb_check_box = gr.Checkbox( - label=translations["reverb"], - value=False, - interactive=True - ) - chorus_check_box = gr.Checkbox( - label=translations["chorus"], - value=False, - interactive=True - ) - delay_check_box = gr.Checkbox( - label=translations["delay"], - value=False, - interactive=True - ) - phaser_check_box = gr.Checkbox( - label=translations["phaser"], - value=False, - interactive=True - ) - compressor_check_box = gr.Checkbox( - label=translations["compressor"], - value=False, - interactive=True - ) - more_options = gr.Checkbox( - label=translations["more_option"], - value=False, - interactive=True - ) - with gr.Row(): - with gr.Accordion( - translations["input_output"], - open=False - ): - with gr.Row(): - upload_audio_files = gr.Files( - label=translations["drop_audio"], - file_types=file_types - ) - with gr.Row(): - audio_in_path = gr.Dropdown( - label=translations["input_audio"], - value="", - choices=paths_for_files, - info=translations["provide_audio"], - interactive=True, - allow_custom_value=True - ) - audio_out_path = gr.Textbox( - label=translations["output_audio"], - value="audios/audio_effects.wav", - placeholder="audios/audio_effects.wav", - info=translations["provide_output"], - interactive=True - ) - with gr.Row(): - with gr.Column(): - audio_combination = gr.Checkbox( - label=translations["merge_instruments"], - value=False, - interactive=True - ) - audio_combination_input = gr.Dropdown( - label=translations["input_audio"], - value="", - choices=paths_for_files, - info=translations["provide_audio"], - interactive=True, - allow_custom_value=True, - visible=False - ) - with gr.Row(): - main_volume = gr.Slider( - minimum=-80, - maximum=80, - label=translations["main_volume"], - info=translations["main_volume_info"], - value=-4, - step=1, - interactive=True, - visible=False - ) - combine_volume = gr.Slider( - minimum=-80, - maximum=80, - label=translations["combination_volume"], - info=translations["combination_volume_info"], - value=-7, - step=1, - interactive=True, - visible=False - ) - with gr.Row(): - audio_effects_refresh = gr.Button( - translations["refresh"] - ) - with gr.Row(): - audio_output_format = gr.Radio( - label=translations["export_format"], - info=translations["export_info"], - choices=export_format_choices, - value="wav", - interactive=True - ) - with gr.Row(): - with gr.Accordion( - translations["use_presets"], - open=False - ): - with gr.Row(): - presets_name = gr.Dropdown( - label=translations["file_preset"], - choices=audio_effect_presets_file, - value=( - audio_effect_presets_file[0] - ) if len(audio_effect_presets_file) > 0 else "", - interactive=True, - allow_custom_value=True - ) - with gr.Row(): - load_preset_button = gr.Button( - translations["load_file"], - variant="primary" - ) - refresh_preset_button = gr.Button( - translations["refresh"] - ) - with gr.Accordion( - translations["export_file"], - open=False - ): - with gr.Row(): - with gr.Column(): - name_to_save_file = gr.Textbox( - label=translations["filename_to_save"] - ) - save_file_button = gr.Button( - translations["export_file"] - ) - with gr.Row(): - upload_presets = gr.Files( - label=translations["upload_presets"], - file_types=[".effect.json"] - ) - with gr.Row(): - apply_effects_button = gr.Button( - translations["apply"], - variant="primary", - scale=2 - ) - with gr.Accordion(translations["audio_effects"], visible=False) as accordion: - with gr.Row(): - with gr.Accordion( - translations["reverb"], - open=True, - visible=False - ) as reverb_accordion: - reverb_freeze_mode = gr.Checkbox( - label=translations["reverb_freeze"], - info=translations["reverb_freeze_info"], - value=False, - interactive=True - ) - reverb_room_size = gr.Slider( - minimum=0, - maximum=1, - step=0.01, - value=0.15, - label=translations["room_size"], - info=translations["room_size_info"], - interactive=True - ) - reverb_damping = gr.Slider( - minimum=0, - maximum=1, - step=0.01, - value=0.7, - label=translations["damping"], - info=translations["damping_info"], - interactive=True - ) - reverb_wet_level = gr.Slider( - minimum=0, - maximum=1, - step=0.01, - value=0.2, - label=translations["wet_level"], - info=translations["wet_level_info"], - interactive=True - ) - reverb_dry_level = gr.Slider( - minimum=0, - maximum=1, - step=0.01, - value=0.8, - label=translations["dry_level"], - info=translations["dry_level_info"], - interactive=True - ) - reverb_width = gr.Slider( - minimum=0, - maximum=1, - step=0.01, - value=1, - label=translations["width"], - info=translations["width_info"], - interactive=True - ) - with gr.Row(): - with gr.Accordion( - translations["chorus"], - open=True, - visible=False - ) as chorus_accordion: - chorus_depth = gr.Slider( - minimum=0, - maximum=1, - step=0.01, - value=0.5, - label=translations["chorus_depth"], - info=translations["chorus_depth_info"], - interactive=True - ) - chorus_rate_hz = gr.Slider( - minimum=0.1, - maximum=10, - step=0.1, - value=1.5, - label=translations["chorus_rate_hz"], - info=translations["chorus_rate_hz_info"], - interactive=True - ) - chorus_mix = gr.Slider( - minimum=0, - maximum=1, - step=0.01, - value=0.5, - label=translations["chorus_mix"], - info=translations["chorus_mix_info"], - interactive=True - ) - chorus_center_delay_ms = gr.Slider( - minimum=0, - maximum=50, - step=1, - value=10, - label=translations["chorus_center_delay_ms"], - info=translations["chorus_center_delay_ms_info"], - interactive=True - ) - chorus_feedback = gr.Slider( - minimum=-1, - maximum=1, - step=0.01, - value=0, - label=translations["chorus_feedback"], - info=translations["chorus_feedback_info"], - interactive=True - ) - with gr.Row(): - with gr.Accordion( - translations["delay"], - open=True, - visible=False - ) as delay_accordion: - delay_second = gr.Slider( - minimum=0, - maximum=5, - step=0.01, - value=0.5, - label=translations["delay_seconds"], - info=translations["delay_seconds_info"], - interactive=True - ) - delay_feedback = gr.Slider( - minimum=0, - maximum=1, - step=0.01, - value=0.5, - label=translations["delay_feedback"], - info=translations["delay_feedback_info"], - interactive=True - ) - delay_mix = gr.Slider( - minimum=0, - maximum=1, - step=0.01, - value=0.5, - label=translations["delay_mix"], - info=translations["delay_mix_info"], - interactive=True - ) - with gr.Row(): - with gr.Accordion( - translations["phaser"], - open=True, - visible=False - ) as phaser_accordion: - phaser_depth = gr.Slider( - minimum=0, - maximum=1, - step=0.01, - value=0.5, - label=translations["phaser_depth"], - info=translations["phaser_depth_info"], - interactive=True - ) - phaser_rate_hz = gr.Slider( - minimum=0.1, - maximum=10, - step=0.1, - value=1, - label=translations["phaser_rate_hz"], - info=translations["phaser_rate_hz_info"], - interactive=True - ) - phaser_mix = gr.Slider( - minimum=0, - maximum=1, - step=0.01, - value=0.5, - label=translations["phaser_mix"], - info=translations["phaser_mix_info"], - interactive=True - ) - phaser_centre_frequency_hz = gr.Slider( - minimum=50, - maximum=5000, - step=10, - value=1000, - label=translations["phaser_centre_frequency_hz"], - info=translations["phaser_centre_frequency_hz_info"], - interactive=True - ) - phaser_feedback = gr.Slider( - minimum=-1, - maximum=1, - step=0.01, - value=0, - label=translations["phaser_feedback"], - info=translations["phaser_feedback_info"], - interactive=True - ) - with gr.Row(): - with gr.Accordion( - translations["compressor"], - open=True, - visible=False - ) as compressor_accordion: - compressor_threshold_db = gr.Slider( - minimum=-60, - maximum=0, - step=1, - value=-20, - label=translations["compressor_threshold_db"], - info=translations["compressor_threshold_db_info"], - interactive=True - ) - compressor_ratio = gr.Slider( - minimum=1, - maximum=20, - step=0.1, - value=1, - label=translations["compressor_ratio"], - info=translations["compressor_ratio_info"], - interactive=True - ) - compressor_attack_ms = gr.Slider( - minimum=0.1, - maximum=100, - step=0.1, - value=10, - label=translations["compressor_attack_ms"], - info=translations["compressor_attack_ms_info"], - interactive=True - ) - compressor_release_ms = gr.Slider( - minimum=10, - maximum=1000, - step=1, - value=100, - label=translations["compressor_release_ms"], - info=translations["compressor_release_ms_info"], - interactive=True - ) - with gr.Row(): - with gr.Accordion( - translations["more_option"], - open=True, - visible=False - ) as more_accordion: - with gr.Row(): - fade = gr.Checkbox( - label=translations["fade"], - value=False, - interactive=True - ) - bass_or_treble = gr.Checkbox( - label=translations["bass_or_treble"], - value=False, - interactive=True - ) - limiter = gr.Checkbox( - label=translations["limiter"], - value=False, - interactive=True - ) - resample_checkbox = gr.Checkbox( - label=translations["resample"], - value=False, - interactive=True - ) - with gr.Row(): - distortion_checkbox = gr.Checkbox( - label=translations["distortion"], - value=False, - interactive=True - ) - gain_checkbox = gr.Checkbox( - label=translations["gain"], - value=False, - interactive=True - ) - bitcrush_checkbox = gr.Checkbox( - label=translations["bitcrush"], - value=False, - interactive=True - ) - clipping_checkbox = gr.Checkbox( - label=translations["clipping"], - value=False, - interactive=True - ) - with gr.Accordion( - translations["fade"], - open=True, - visible=False - ) as fade_accordion: - with gr.Row(): - fade_in = gr.Slider( - minimum=0, - maximum=10000, - step=100, - value=0, - label=translations["fade_in"], - info=translations["fade_in_info"], - interactive=True - ) - fade_out = gr.Slider( - minimum=0, - maximum=10000, - step=100, - value=0, - label=translations["fade_out"], - info=translations["fade_out_info"], - interactive=True - ) - with gr.Accordion( - translations["bass_or_treble"], - open=True, - visible=False - ) as bass_treble_accordion: - with gr.Row(): - bass_boost = gr.Slider( - minimum=0, - maximum=20, - step=1, - value=0, - label=translations["bass_boost"], - info=translations["bass_boost_info"], - interactive=True - ) - bass_frequency = gr.Slider( - minimum=20, - maximum=200, - step=10, - value=100, - label=translations["bass_frequency"], - info=translations["bass_frequency_info"], - interactive=True - ) - with gr.Row(): - treble_boost = gr.Slider( - minimum=0, - maximum=20, - step=1, - value=0, - label=translations["treble_boost"], - info=translations["treble_boost_info"], - interactive=True - ) - treble_frequency = gr.Slider( - minimum=1000, - maximum=10000, - step=500, - value=3000, - label=translations["treble_frequency"], - info=translations["treble_frequency_info"], - interactive=True - ) - with gr.Accordion( - translations["limiter"], - open=True, - visible=False - ) as limiter_accordion: - with gr.Row(): - limiter_threshold_db = gr.Slider( - minimum=-60, - maximum=0, - step=1, - value=-1, - label=translations["limiter_threshold_db"], - info=translations["limiter_threshold_db_info"], - interactive=True - ) - limiter_release_ms = gr.Slider( - minimum=10, - maximum=1000, - step=1, - value=100, - label=translations["limiter_release_ms"], - info=translations["limiter_release_ms_info"], - interactive=True - ) - with gr.Column(): - pitch_shift_semitones = gr.Slider( - minimum=-20, - maximum=20, - step=1, - value=0, - label=translations["pitch"], - info=translations["pitch_info"], - interactive=True - ) - audio_effect_resample_sr = gr.Radio( - choices=[0]+sample_rate_choice, - value=0, - label=translations["resample"], - info=translations["resample_info"], - interactive=True, - visible=False - ) - distortion_drive_db = gr.Slider( - minimum=0, - maximum=50, - step=1, - value=20, - label=translations["distortion"], - info=translations["distortion_info"], - interactive=True, - visible=False - ) - gain_db = gr.Slider( - minimum=-60, - maximum=60, - step=1, - value=0, - label=translations["gain"], - info=translations["gain_info"], - interactive=True, - visible=False - ) - clipping_threshold_db = gr.Slider( - minimum=-60, - maximum=0, - step=1, - value=-1, - label=translations["clipping_threshold_db"], - info=translations["clipping_threshold_db_info"], - interactive=True, - visible=False - ) - bitcrush_bit_depth = gr.Slider( - minimum=1, - maximum=24, - step=1, - value=16, - label=translations["bitcrush_bit_depth"], - info=translations["bitcrush_bit_depth_info"], - interactive=True, - visible=False - ) - with gr.Row(): - gr.Markdown(translations["output_audio"]) - with gr.Row(): - audio_play_input = gr.Audio( - interactive=False, - label=translations["input_audio"], - **audio_params - ) - audio_play_output = gr.Audio( - interactive=False, - label=translations["output_audio"], - **audio_params - ) - with gr.Row(): - reverb_check_box.change( - fn=visible, - inputs=[ - reverb_check_box - ], - outputs=[ - reverb_accordion - ] - ) - chorus_check_box.change( - fn=visible, - inputs=[ - chorus_check_box - ], - outputs=[ - chorus_accordion - ] - ) - delay_check_box.change( - fn=visible, - inputs=[ - delay_check_box - ], - outputs=[ - delay_accordion - ] - ) - with gr.Row(): - compressor_check_box.change( - fn=visible, - inputs=[ - compressor_check_box - ], - outputs=[ - compressor_accordion - ] - ) - phaser_check_box.change( - fn=visible, - inputs=[ - phaser_check_box - ], - outputs=[ - phaser_accordion - ] - ) - more_options.change( - fn=visible, - inputs=[ - more_options - ], - outputs=[ - more_accordion - ] - ) - with gr.Row(): - fade.change( - fn=visible, - inputs=[ - fade - ], - outputs=[ - fade_accordion - ] - ) - bass_or_treble.change( - fn=visible, - inputs=[ - bass_or_treble - ], - outputs=[ - bass_treble_accordion - ] - ) - limiter.change( - fn=visible, - inputs=[ - limiter - ], - outputs=[ - limiter_accordion - ] - ) - resample_checkbox.change( - fn=visible, - inputs=[ - resample_checkbox - ], - outputs=[ - audio_effect_resample_sr - ] - ) - with gr.Row(): - distortion_checkbox.change( - fn=visible, - inputs=[ - distortion_checkbox - ], - outputs=[ - distortion_drive_db - ] - ) - gain_checkbox.change( - fn=visible, - inputs=[ - gain_checkbox - ], - outputs=[ - gain_db - ] - ) - clipping_checkbox.change( - fn=visible, - inputs=[ - clipping_checkbox - ], - outputs=[ - clipping_threshold_db - ] - ) - with gr.Row(): - bitcrush_checkbox.change( - fn=visible, - inputs=[ - bitcrush_checkbox - ], - outputs=[ - bitcrush_bit_depth - ] - ) - upload_audio_files.upload( - fn=lambda audio_in: [ - shutil_move(audio.name, configs["audios_path"]) - for audio in audio_in - ][0], - inputs=[ - upload_audio_files - ], - outputs=[ - audio_in_path - ] - ) - audio_in_path.change( - fn=lambda audio: audio if audio else None, - inputs=[ - audio_in_path - ], - outputs=[ - audio_play_input - ] - ) - with gr.Row(): - audio_effects_refresh.click( - fn=lambda a, b: [ - change_audios_choices(a), - change_audios_choices(b) - ], - inputs=[ - audio_in_path, - audio_combination_input - ], - outputs=[ - audio_in_path, - audio_combination_input - ] - ) - audio_combination.change( - fn=visible, - inputs=[ - audio_combination - ], - outputs=[ - audio_combination_input - ] - ) - audio_combination.change( - fn=lambda a: [ - visible(a) - for _ in range(2) - ], - inputs=[ - audio_combination - ], - outputs=[ - main_volume, - combine_volume - ] - ) - with gr.Row(): - more_options.change( - fn=lambda: [False]*8, - inputs=[], - outputs=[ - fade, - bass_or_treble, - limiter, - resample_checkbox, - distortion_checkbox, - gain_checkbox, - clipping_checkbox, - bitcrush_checkbox - ] - ) - upload_presets.upload( - fn=lambda presets_in: [ - shutil_move(preset.name, configs["presets_path"]) - for preset in presets_in - ][0], - inputs=[ - upload_presets - ], - outputs=[ - presets_name - ] - ) - refresh_preset_button.click( - fn=change_effect_preset_choices, - inputs=[], - outputs=[ - presets_name - ] - ) - with gr.Row(): - reverb_check_box.change( - fn=lambda a, b, c, d, e, f: visible(max([a, b, c, d, e, f])), - inputs=[ - reverb_check_box, - chorus_check_box, - delay_check_box, - phaser_check_box, - compressor_check_box, - more_options - ], - outputs=[accordion] - ) - chorus_check_box.change( - fn=lambda a, b, c, d, e, f: visible(max([a, b, c, d, e, f])), - inputs=[ - reverb_check_box, - chorus_check_box, - delay_check_box, - phaser_check_box, - compressor_check_box, - more_options - ], - outputs=[accordion] - ) - delay_check_box.change( - fn=lambda a, b, c, d, e, f: visible(max([a, b, c, d, e, f])), - inputs=[ - reverb_check_box, - chorus_check_box, - delay_check_box, - phaser_check_box, - compressor_check_box, - more_options - ], - outputs=[accordion] - ) - with gr.Row(): - phaser_check_box.change( - fn=lambda a, b, c, d, e, f: visible(max([a, b, c, d, e, f])), - inputs=[ - reverb_check_box, - chorus_check_box, - delay_check_box, - phaser_check_box, - compressor_check_box, - more_options - ], - outputs=[accordion] - ) - compressor_check_box.change( - fn=lambda a, b, c, d, e, f: visible(max([a, b, c, d, e, f])), - inputs=[ - reverb_check_box, - chorus_check_box, - delay_check_box, - phaser_check_box, - compressor_check_box, - more_options - ], - outputs=[accordion] - ) - more_options.change( - fn=lambda a, b, c, d, e, f: visible(max([a, b, c, d, e, f])), - inputs=[ - reverb_check_box, - chorus_check_box, - delay_check_box, - phaser_check_box, - compressor_check_box, - more_options - ], - outputs=[accordion] - ) - with gr.Row(): - load_preset_button.click( - fn=audio_effect_load_presets, - inputs=[ - presets_name, - resample_checkbox, - audio_effect_resample_sr, - chorus_depth, - chorus_rate_hz, - chorus_mix, - chorus_center_delay_ms, - chorus_feedback, - distortion_drive_db, - reverb_room_size, - reverb_damping, - reverb_wet_level, - reverb_dry_level, - reverb_width, - reverb_freeze_mode, - pitch_shift_semitones, - delay_second, - delay_feedback, - delay_mix, - compressor_threshold_db, - compressor_ratio, - compressor_attack_ms, - compressor_release_ms, - limiter_threshold_db, - limiter_release_ms, - gain_db, - bitcrush_bit_depth, - clipping_threshold_db, - phaser_rate_hz, - phaser_depth, - phaser_centre_frequency_hz, - phaser_feedback, - phaser_mix, - bass_boost, - bass_frequency, - treble_boost, - treble_frequency, - fade_in, - fade_out, - chorus_check_box, - distortion_checkbox, - reverb_check_box, - delay_check_box, - compressor_check_box, - limiter, - gain_checkbox, - bitcrush_checkbox, - clipping_checkbox, - phaser_check_box, - bass_or_treble, - fade - ], - outputs=[ - resample_checkbox, - audio_effect_resample_sr, - chorus_depth, - chorus_rate_hz, - chorus_mix, - chorus_center_delay_ms, - chorus_feedback, - distortion_drive_db, - reverb_room_size, - reverb_damping, - reverb_wet_level, - reverb_dry_level, - reverb_width, - reverb_freeze_mode, - pitch_shift_semitones, - delay_second, - delay_feedback, - delay_mix, - compressor_threshold_db, - compressor_ratio, - compressor_attack_ms, - compressor_release_ms, - limiter_threshold_db, - limiter_release_ms, - gain_db, - bitcrush_bit_depth, - clipping_threshold_db, - phaser_rate_hz, - phaser_depth, - phaser_centre_frequency_hz, - phaser_feedback, - phaser_mix, - bass_boost, - bass_frequency, - treble_boost, - treble_frequency, - fade_in, - fade_out, - chorus_check_box, - distortion_checkbox, - reverb_check_box, - delay_check_box, - compressor_check_box, - limiter, - gain_checkbox, - bitcrush_checkbox, - clipping_checkbox, - phaser_check_box, - bass_or_treble, - fade - ], - ) - save_file_button.click( - fn=audio_effect_save_presets, - inputs=[ - name_to_save_file, - resample_checkbox, - audio_effect_resample_sr, - chorus_depth, - chorus_rate_hz, - chorus_mix, - chorus_center_delay_ms, - chorus_feedback, - distortion_drive_db, - reverb_room_size, - reverb_damping, - reverb_wet_level, - reverb_dry_level, - reverb_width, - reverb_freeze_mode, - pitch_shift_semitones, - delay_second, - delay_feedback, - delay_mix, - compressor_threshold_db, - compressor_ratio, - compressor_attack_ms, - compressor_release_ms, - limiter_threshold_db, - limiter_release_ms, - gain_db, - bitcrush_bit_depth, - clipping_threshold_db, - phaser_rate_hz, - phaser_depth, - phaser_centre_frequency_hz, - phaser_feedback, - phaser_mix, - bass_boost, - bass_frequency, - treble_boost, - treble_frequency, - fade_in, - fade_out, - chorus_check_box, - distortion_checkbox, - reverb_check_box, - delay_check_box, - compressor_check_box, - limiter, - gain_checkbox, - bitcrush_checkbox, - clipping_checkbox, - phaser_check_box, - bass_or_treble, - fade - ], - outputs=[ - presets_name - ] - ) - apply_effects_button.click( - fn=audio_effects, - inputs=[ - audio_in_path, - audio_out_path, - resample_checkbox, - audio_effect_resample_sr, - chorus_depth, - chorus_rate_hz, - chorus_mix, - chorus_center_delay_ms, - chorus_feedback, - distortion_drive_db, - reverb_room_size, - reverb_damping, - reverb_wet_level, - reverb_dry_level, - reverb_width, - reverb_freeze_mode, - pitch_shift_semitones, - delay_second, - delay_feedback, - delay_mix, - compressor_threshold_db, - compressor_ratio, - compressor_attack_ms, - compressor_release_ms, - limiter_threshold_db, - limiter_release_ms, - gain_db, - bitcrush_bit_depth, - clipping_threshold_db, - phaser_rate_hz, - phaser_depth, - phaser_centre_frequency_hz, - phaser_feedback, - phaser_mix, - bass_boost, - bass_frequency, - treble_boost, - treble_frequency, - fade_in, - fade_out, - audio_output_format, - chorus_check_box, - distortion_checkbox, - reverb_check_box, - delay_check_box, - compressor_check_box, - limiter, - gain_checkbox, - bitcrush_checkbox, - clipping_checkbox, - phaser_check_box, - bass_or_treble, - fade, - audio_combination, - audio_combination_input, - main_volume, - combine_volume - ], - outputs=[ - audio_play_output - ], - api_name="audio_effects" - ) \ No newline at end of file diff --git a/main/app/tabs/editing/child/quirk.py b/main/app/tabs/editing/child/quirk.py deleted file mode 100644 index d010d8fb9dfc74ca2a244e711623a238985fdf11..0000000000000000000000000000000000000000 --- a/main/app/tabs/editing/child/quirk.py +++ /dev/null @@ -1,130 +0,0 @@ -import os -import sys - -import gradio as gr - -sys.path.append(os.getcwd()) - -from main.app.core.editing import apply_voice_quirk - -from main.app.core.ui import ( - shutil_move, - change_audios_choices -) - -from main.app.variables import ( - configs, - file_types, - audio_params, - translations, - paths_for_files, - export_format_choices -) - -def quirk_tab(): - with gr.Row(): - gr.Markdown(translations["quirk_markdown"]) - with gr.Row(): - input_audio_play = gr.Audio( - interactive=False, - label=translations["input_audio"], - **audio_params - ) - with gr.Row(): - quirk_choice = gr.Radio( - label=translations["quirk_label"], - info=translations["quirk_label_info"], - choices=list(translations["quirk_choice"].keys()), - interactive=True, - value=list(translations["quirk_choice"].keys())[0] - ) - with gr.Row(): - apply_quirk_button = gr.Button( - translations["apply"], - variant="primary" - ) - with gr.Row(): - with gr.Accordion(translations["input_output"], open=False): - with gr.Row(): - quirk_upload_audio = gr.Files( - label=translations["drop_audio"], - file_types=file_types - ) - with gr.Column(): - quirk_export_format = gr.Radio( - label=translations["export_format"], - info=translations["export_info"], - choices=export_format_choices, - value="wav", - interactive=True - ) - quirk_input_path = gr.Dropdown( - label=translations["audio_path"], - value="", - choices=paths_for_files, - info=translations["provide_audio"], - allow_custom_value=True, - interactive=True - ) - quirk_output_path = gr.Textbox( - label=translations["output_path"], - value="audios/output.wav", - placeholder="audios/output.wav", - info=translations["output_path_info"], - interactive=True - ) - with gr.Column(): - quirk_refresh = gr.Button( - translations["refresh"] - ) - with gr.Row(): - output_audio_play = gr.Audio( - interactive=False, - label=translations["output_audio"], - **audio_params - ) - with gr.Row(): - quirk_upload_audio.upload( - fn=lambda audio_in: [ - shutil_move(audio.name, configs["audios_path"]) - for audio in audio_in - ][0], - inputs=[ - quirk_upload_audio - ], - outputs=[ - quirk_input_path - ] - ) - quirk_input_path.change( - fn=lambda audio: audio if audio else None, - inputs=[ - quirk_input_path - ], - outputs=[ - input_audio_play - ] - ) - quirk_refresh.click( - fn=change_audios_choices, - inputs=[ - quirk_input_path - ], - outputs=[ - quirk_input_path - ] - ) - with gr.Row(): - apply_quirk_button.click( - fn=apply_voice_quirk, - inputs=[ - quirk_input_path, - quirk_choice, - quirk_output_path, - quirk_export_format - ], - outputs=[ - output_audio_play - ], - api_name="quirk" - ) \ No newline at end of file diff --git a/main/app/tabs/editing/editing.py b/main/app/tabs/editing/editing.py deleted file mode 100644 index bb631732c7be16d6b2875ceb3d2e2aa8d69af4d4..0000000000000000000000000000000000000000 --- a/main/app/tabs/editing/editing.py +++ /dev/null @@ -1,27 +0,0 @@ -import os -import sys - -import gradio as gr - -sys.path.append(os.getcwd()) - -from main.app.variables import ( - configs, - translations -) - -def editing_tab(): - with gr.TabItem(translations["editing"]): - if configs.get("effects_tab", True): - from main.app.tabs.editing.child.audio_effects import audio_effects_tab - - with gr.TabItem(translations["audio_effects"]): - gr.Markdown(translations["apply_audio_effects"]) - audio_effects_tab() - - if configs.get("quirk_tab", True): - from main.app.tabs.editing.child.quirk import quirk_tab - - with gr.TabItem(translations["quirk"]): - gr.Markdown(translations["quirk_info"]) - quirk_tab() \ No newline at end of file diff --git a/main/app/tabs/extra/child/analyzer.py b/main/app/tabs/extra/child/analyzer.py deleted file mode 100644 index e7b851dc2a4c9f92c6b81b3a6f64056fb7dd4c4a..0000000000000000000000000000000000000000 --- a/main/app/tabs/extra/child/analyzer.py +++ /dev/null @@ -1,94 +0,0 @@ -import os -import sys - -import gradio as gr - -sys.path.append(os.getcwd()) - -from main.tools.analyzer import analyze_audio -from main.app.core.ui import change_audios_choices, shutil_move -from main.app.variables import translations, paths_for_files, audio_params, file_types, configs - -def analyzer_tab(): - with gr.Row(): - gr.Markdown(translations["analyzer_markdown_2"]) - with gr.Row(): - with gr.Accordion( - translations["input_output"], - open=False - ): - input_audio = gr.Dropdown( - label=translations["audio_path"], - value="", - choices=paths_for_files, - info=translations["provide_audio"], - allow_custom_value=True, - interactive=True - ) - upload_audio_file = gr.Files( - label=translations["drop_audio"], - file_types=file_types - ) - with gr.Row(): - play_audio = gr.Audio( - interactive=False, - label=translations["input_audio"], - **audio_params - ) - with gr.Row(): - refresh_audio = gr.Button( - translations["refresh"] - ) - with gr.Row(): - get_info_button = gr.Button( - value=translations["read_audio"], - variant="primary" - ) - with gr.Row(): - output_info = gr.Textbox( - label=translations["output_information"], - value="", - max_lines=8, - interactive=False, - ) - with gr.Row(): - image_output = gr.Image( - interactive=False - ) - with gr.Row(): - get_info_button.click( - fn=analyze_audio, - inputs=[input_audio], - outputs=[output_info, image_output], - ) - input_audio.change( - fn=lambda audio: audio if os.path.isfile(audio) else None, - inputs=[ - input_audio - ], - outputs=[ - play_audio - ] - ) - refresh_audio.click( - fn=change_audios_choices, - inputs=[ - input_audio - ], - outputs=[ - input_audio - ] - ) - with gr.Row(): - upload_audio_file.upload( - fn=lambda audio_in: [ - shutil_move(audio.name, configs["audios_path"]) - for audio in audio_in - ][0], - inputs=[ - upload_audio_file - ], - outputs=[ - input_audio - ] - ) \ No newline at end of file diff --git a/main/app/tabs/extra/child/convert_model_onnx.py b/main/app/tabs/extra/child/convert_model_onnx.py deleted file mode 100644 index b49f49180aa53defd1f3f95048dcaffbff2e1572..0000000000000000000000000000000000000000 --- a/main/app/tabs/extra/child/convert_model_onnx.py +++ /dev/null @@ -1,93 +0,0 @@ -import os -import sys - -import gradio as gr - -sys.path.append(os.getcwd()) - -from main.app.core.model_utils import onnx_export - -from main.app.core.ui import ( - visible, - shutil_move, - change_models_choices -) - -from main.app.variables import ( - configs, - model_name, - translations -) - -def convert_onnx_model_tab(): - with gr.Row(): - gr.Markdown(translations["pytorch2onnx_markdown"]) - with gr.Row(): - model_pth_upload = gr.File( - label=translations["drop_model"], - file_types=[".pth"] - ) - with gr.Accordion( - label=translations["model_name"], - open=True - ): - with gr.Row(): - model_pth_path = gr.Dropdown( - label=translations["model_name"], - choices=model_name, - value=model_name[0] if len(model_name) >= 1 else "", - interactive=True, - allow_custom_value=True - ) - with gr.Row(): - refresh_model = gr.Button( - translations["refresh"] - ) - with gr.Row(): - convert_onnx_button = gr.Button( - translations["convert_model"], - variant="primary", - scale=2 - ) - with gr.Row(): - output_model_file = gr.File( - label=translations["output_model_path"], - file_types=[".onnx"], - interactive=False, - visible=False - ) - with gr.Row(): - model_pth_upload.upload( - fn=lambda model_pth_upload: shutil_move(model_pth_upload.name, configs["weights_path"]), - inputs=[ - model_pth_upload - ], - outputs=[ - model_pth_path - ] - ) - refresh_model.click( - fn=lambda: change_models_choices()[0], - inputs=[], - outputs=[ - model_pth_path - ] - ) - with gr.Row(): - convert_onnx_button.click( - fn=onnx_export, - inputs=[ - model_pth_path - ], - outputs=[ - output_model_file - ], - api_name="model_onnx_export" - ) - convert_onnx_button.click( - fn=lambda: visible(True), - inputs=[], - outputs=[ - output_model_file - ] - ) \ No newline at end of file diff --git a/main/app/tabs/extra/child/convert_model_svc.py b/main/app/tabs/extra/child/convert_model_svc.py deleted file mode 100644 index b405affbf565df8acf8ce554a0dc14762804e58c..0000000000000000000000000000000000000000 --- a/main/app/tabs/extra/child/convert_model_svc.py +++ /dev/null @@ -1,128 +0,0 @@ -import os -import sys - -import gradio as gr - -sys.path.append(os.getcwd()) - -from main.app.core.model_utils import svc_export - -from main.app.core.ui import ( - visible, - shutil_move, - change_models_choices -) - -from main.app.variables import ( - configs, - model_name, - translations -) - -def convert_svc_model_tab(): - with gr.Row(): - gr.Markdown(translations["convert_model_svc_markdown_2"]) - with gr.Row(): - delete_when_success = gr.Checkbox( - label=translations["delete_when_success"], - value=True, - interactive=True - ) - with gr.Row(): - model_pth_upload = gr.File( - label=translations["drop_model"], - file_types=[".pth"] - ) - config_upload = gr.File( - label=translations["drop_json"], - file_types=[".json"] - ) - with gr.Accordion( - label=translations["model_name"], - open=True - ): - with gr.Row(): - model_pth_path = gr.Dropdown( - label=translations["model_name"], - choices=model_name, - value=model_name[0] if len(model_name) >= 1 else "", - interactive=True, - allow_custom_value=True - ) - config_path = gr.Textbox( - label=translations["config_path"], - value="", - placeholder="config.json", - interactive=True - ) - with gr.Row(): - modelname = gr.Textbox( - label=translations["modelname"], - value="", - placeholder=translations["modelname"], - interactive=True - ) - with gr.Row(): - refresh_model = gr.Button( - translations["refresh"] - ) - with gr.Row(): - convert_svc_button = gr.Button( - translations["convert_model_svc"], - variant="primary", - scale=2 - ) - with gr.Row(): - output_model_file = gr.File( - label=translations["output_model_path"], - file_types=[".pth", ".onnx"], - interactive=False, - visible=False - ) - with gr.Row(): - model_pth_upload.upload( - fn=lambda model_pth_upload: shutil_move(model_pth_upload.name, configs["weights_path"]), - inputs=[ - model_pth_upload - ], - outputs=[ - model_pth_path - ] - ) - config_upload.upload( - fn=lambda config: shutil_move(config.name, configs["weights_path"]), - inputs=[ - config_upload - ], - outputs=[ - config_path - ] - ) - refresh_model.click( - fn=lambda: change_models_choices()[0], - inputs=[], - outputs=[ - model_pth_path - ] - ) - with gr.Row(): - convert_svc_button.click( - fn=svc_export, - inputs=[ - model_pth_path, - config_path, - modelname, - delete_when_success - ], - outputs=[ - output_model_file - ], - api_name="model_svc_export" - ) - convert_svc_button.click( - fn=lambda: visible(True), - inputs=[], - outputs=[ - output_model_file - ] - ) \ No newline at end of file diff --git a/main/app/tabs/extra/child/create_srt.py b/main/app/tabs/extra/child/create_srt.py deleted file mode 100644 index 550a00566824da035fcb3a8cc015123cd9c74754..0000000000000000000000000000000000000000 --- a/main/app/tabs/extra/child/create_srt.py +++ /dev/null @@ -1,145 +0,0 @@ -import os -import sys - -import gradio as gr - -sys.path.append(os.getcwd()) - -from main.app.core.csrt import create_srt - -from main.app.core.ui import ( - shutil_move, - change_audios_choices -) - -from main.app.variables import ( - configs, - file_types, - translations, - audio_params, - whisper_model, - paths_for_files, - whisper_languages -) - -def create_srt_tab(): - with gr.Row(): - gr.Markdown(translations["create_srt_markdown_2"]) - with gr.Row(): - with gr.Column(): - srt_content = gr.Textbox( - label=translations["srt_content"], - value="", - lines=11, - max_lines=11, - interactive=False - ) - with gr.Column(): - with gr.Group(): - language = gr.Dropdown( - label=translations["whisper_languages"], - value="vi", - choices=whisper_languages, - info=translations["whisper_languages_info"], - interactive=True - ) - model_size = gr.Radio( - label=translations["model_size"], - info=translations["model_size_info"], - choices=whisper_model, - value=whisper_model[0], - interactive=True - ) - with gr.Row(): - convert_button = gr.Button( - translations["convert_audio"], - variant="primary" - ) - with gr.Row(): - with gr.Accordion( - translations["input_output"], - open=False - ): - with gr.Column(): - input_audio = gr.Dropdown( - label=translations["audio_path"], - value="", - choices=paths_for_files, - info=translations["provide_audio"], - allow_custom_value=True, - interactive=True - ) - output_file = gr.Textbox( - label=translations["srt_path_output_file"], - value="srt/output.srt", - placeholder="srt/output.srt", - interactive=True - ) - with gr.Column(): - refresh = gr.Button( - translations["refresh"] - ) - with gr.Row(): - upload_audio_files = gr.Files( - label=translations["drop_audio"], - file_types=file_types - ) - with gr.Row(): - play_audio = gr.Audio( - interactive=False, - label=translations["input_audio"], - **audio_params - ) - with gr.Row(): - output_srt = gr.File( - label=translations["srt_output_file"], - file_types=[".srt"], - interactive=False, - visible=False - ) - with gr.Row(): - upload_audio_files.upload( - fn=lambda audio_in: [ - shutil_move(audio.name, configs["audios_path"]) - for audio in audio_in - ][0], - inputs=[ - upload_audio_files - ], - outputs=[ - input_audio - ] - ) - input_audio.change( - fn=lambda audio: audio if os.path.isfile(audio) else None, - inputs=[ - input_audio - ], - outputs=[ - play_audio - ] - ) - refresh.click( - fn=change_audios_choices, - inputs=[ - input_audio - ], - outputs=[ - input_audio - ] - ) - with gr.Row(): - convert_button.click( - fn=create_srt, - inputs=[ - model_size, - input_audio, - output_file, - language - ], - outputs=[ - output_srt, - srt_content - ], - api_name="create_srt" - ) \ No newline at end of file diff --git a/main/app/tabs/extra/child/f0_extract.py b/main/app/tabs/extra/child/f0_extract.py deleted file mode 100644 index 7cfa1307d40f30c362c95d077091479b25cd832e..0000000000000000000000000000000000000000 --- a/main/app/tabs/extra/child/f0_extract.py +++ /dev/null @@ -1,225 +0,0 @@ -import os -import sys - -import gradio as gr - -sys.path.append(os.getcwd()) - -from main.app.core.f0_extract import f0_extract - -from main.app.core.ui import ( - visible, - unlock_f0, - shutil_move, - change_audios_choices -) - -from main.app.variables import ( - configs, - method_f0, - file_types, - audio_params, - translations, - paths_for_files -) - -def f0_extract_tab(): - with gr.Row(): - gr.Markdown(translations["f0_extractor_markdown_2"]) - with gr.Row(): - extractor_button = gr.Button( - translations["extract_button"].replace("2. ", ""), - variant="primary" - ) - with gr.Row(): - with gr.Column(): - upload_audio_file = gr.Files( - label=translations["drop_audio"], - file_types=file_types - ) - audioplay = gr.Audio( - interactive=False, - label=translations["input_audio"], - **audio_params - ) - with gr.Column(): - with gr.Accordion( - translations["f0_method"], - open=False - ): - with gr.Group(): - with gr.Row(): - predictor_onnx = gr.Checkbox( - label=translations["predictor_onnx"], - value=False, - interactive=True - ) - unlock_full_method = gr.Checkbox( - label=translations["f0_unlock"], - value=False, - interactive=True - ) - autotune = gr.Checkbox( - label=translations["autotune"], - value=False, - interactive=True - ) - proposal_pitch = gr.Checkbox( - label=translations["proposal_pitch"], - value=False, - interactive=True - ) - f0_method_extract = gr.Radio( - label=translations["f0_method"], - info=translations["f0_method_info"], - choices=[m for m in method_f0 if m != "hybrid"], - value="rmvpe", - interactive=True - ) - pitch = gr.Slider( - minimum=-24, - maximum=24, - step=1, - info=translations["pitch_info"], - label=translations["pitch"], - value=0, - interactive=True - ) - f0_autotune_strength = gr.Slider( - minimum=0, - maximum=1, - label=translations["autotune_rate"], - info=translations["autotune_rate_info"], - value=1, - step=0.1, - interactive=True, - visible=False - ) - proposal_pitch_threshold = gr.Slider( - minimum=50.0, - maximum=1200.0, - label=translations["proposal_pitch_threshold"], - info=translations["proposal_pitch_threshold_info"], - value=255.0, - step=0.1, - interactive=True, - visible=False - ) - filter_radius = gr.Slider( - minimum=0, - maximum=7, - label=translations["filter_radius"], - info=translations["filter_radius_info"], - value=3, - step=1, - interactive=True - ) - with gr.Accordion( - translations["audio_path"], - open=True - ): - input_audio_path = gr.Dropdown( - label=translations["audio_path"], - value="", - choices=paths_for_files, - allow_custom_value=True, - interactive=True - ) - refresh_audio_button = gr.Button( - translations["refresh"] - ) - with gr.Row(): - gr.Markdown("___") - with gr.Row(): - file_output = gr.File( - label="", - file_types=[".txt"], - interactive=False - ) - image_output = gr.Image( - label="", - interactive=False - ) - with gr.Row(): - upload_audio_file.upload( - fn=lambda audio_in: [ - shutil_move(audio.name, configs["audios_path"]) - for audio in audio_in - ][0], - inputs=[ - upload_audio_file - ], - outputs=[ - input_audio_path - ] - ) - input_audio_path.change( - fn=lambda audio: audio if os.path.isfile(audio) else None, - inputs=[ - input_audio_path - ], - outputs=[ - audioplay - ] - ) - refresh_audio_button.click( - fn=change_audios_choices, - inputs=[ - input_audio_path - ], - outputs=[ - input_audio_path - ] - ) - with gr.Row(): - autotune.change( - fn=visible, - inputs=[ - autotune - ], - outputs=[ - f0_autotune_strength - ] - ) - proposal_pitch.change( - fn=visible, - inputs=[ - proposal_pitch - ], - outputs=[ - proposal_pitch_threshold - ] - ) - with gr.Row(): - unlock_full_method.change( - fn=lambda method: { - "choices": [m for m in unlock_f0(method)["choices"] if m != "hybrid"], - "value": "rmvpe", - "__type__": "update" - }, - inputs=[ - unlock_full_method - ], - outputs=[ - f0_method_extract - ] - ) - extractor_button.click( - fn=f0_extract, - inputs=[ - input_audio_path, - f0_method_extract, - predictor_onnx, - pitch, - filter_radius, - autotune, - f0_autotune_strength, - proposal_pitch, - proposal_pitch_threshold - ], - outputs=[ - file_output, - image_output - ], - api_name="f0_extract" - ) \ No newline at end of file diff --git a/main/app/tabs/extra/child/fushion.py b/main/app/tabs/extra/child/fushion.py deleted file mode 100644 index 1585930b4e220edb71e3c2bc58eb87b6ffd5951b..0000000000000000000000000000000000000000 --- a/main/app/tabs/extra/child/fushion.py +++ /dev/null @@ -1,132 +0,0 @@ -import os -import sys - -import gradio as gr - -sys.path.append(os.getcwd()) - -from main.app.core.model_utils import fushion_model - -from main.app.core.ui import ( - visible, - shutil_move, - change_models_choices -) - -from main.app.variables import ( - configs, - model_name, - translations -) - -def fushion_tab(): - with gr.Row(): - gr.Markdown(translations["fushion_markdown_2"]) - with gr.Row(): - modelname = gr.Textbox( - label=translations["modelname"], - placeholder="Model.pth", - value="", - max_lines=2, - interactive=True - ) - with gr.Row(): - fushion_button = gr.Button( - translations["fushion"], - variant="primary", - scale=4 - ) - with gr.Column(): - with gr.Row(): - upload_model_1 = gr.File( - label=f"{translations['model_name']} 1", - file_types=[".pth"] - ) - upload_model_2 = gr.File( - label=f"{translations['model_name']} 2", - file_types=[".pth"] - ) - with gr.Row(): - model_path_1 = gr.Dropdown( - label=translations["model_name"] + " 1", - choices=model_name, - value=model_name[0] if len(model_name) >= 1 else "", - interactive=True, - allow_custom_value=True - ) - model_path_2 = gr.Dropdown( - label=translations["model_name"] + " 2", - choices=model_name, - value=model_name[0] if len(model_name) >= 1 else "", - interactive=True, - allow_custom_value=True - ) - with gr.Row(): - refresh_model = gr.Button( - translations["refresh"] - ) - with gr.Row(): - ratio = gr.Slider( - minimum=0, - maximum=1, - label=translations["model_ratio"], - info=translations["model_ratio_info"], - value=0.5, - interactive=True - ) - with gr.Row(): - output_model = gr.File( - label=translations["output_model_path"], - file_types=[".pth", ".onnx"], - interactive=False, - visible=False - ) - with gr.Row(): - upload_model_1.upload( - fn=lambda model: shutil_move(model.name, configs["weights_path"]), - inputs=[ - upload_model_1 - ], - outputs=[ - model_path_1 - ] - ) - upload_model_2.upload( - fn=lambda model: shutil_move(model.name, configs["weights_path"]), - inputs=[ - upload_model_2 - ], - outputs=[ - model_path_2 - ] - ) - refresh_model.click( - fn=lambda: [change_models_choices()[0] for _ in range(2)], - inputs=[], - outputs=[ - model_path_1, - model_path_2 - ] - ) - with gr.Row(): - fushion_button.click( - fn=fushion_model, - inputs=[ - modelname, - model_path_1, - model_path_2, - ratio - ], - outputs=[ - modelname, - output_model - ], - api_name="fushion_model" - ) - fushion_button.click( - fn=lambda: visible(True), - inputs=[], - outputs=[ - output_model - ] - ) \ No newline at end of file diff --git a/main/app/tabs/extra/child/read_model.py b/main/app/tabs/extra/child/read_model.py deleted file mode 100644 index 4a9a298c3c1ad33e7db9d7182f2b9d6fdeb0ff8c..0000000000000000000000000000000000000000 --- a/main/app/tabs/extra/child/read_model.py +++ /dev/null @@ -1,83 +0,0 @@ -import os -import sys - -import gradio as gr - -sys.path.append(os.getcwd()) - -from main.app.core.model_utils import model_info - -from main.app.core.ui import ( - shutil_move, - change_models_choices -) - -from main.app.variables import ( - configs, - model_name, - translations, -) - -def read_model_tab(): - with gr.Row(): - gr.Markdown(translations["read_model_markdown_2"]) - with gr.Row(): - model = gr.File( - label=translations["drop_model"], - file_types=[".pth", ".onnx"] - ) - with gr.Accordion( - label=translations["model_name"], - open=True - ): - model_path = gr.Dropdown( - label=translations["model_name"], - choices=model_name, - value=model_name[0] if len(model_name) >= 1 else "", - interactive=True, - allow_custom_value=True - ) - with gr.Row(): - refresh_model = gr.Button( - translations["refresh"] - ) - with gr.Row(): - read_button = gr.Button( - translations["readmodel"], - variant="primary", - scale=2 - ) - with gr.Row(): - output_info = gr.Textbox( - label=translations["modelinfo"], - value="", - interactive=False, - lines=10 - ) - with gr.Row(): - model.upload( - fn=lambda model: shutil_move(model.name, configs["weights_path"]), - inputs=[ - model - ], - outputs=[ - model_path - ] - ) - read_button.click( - fn=model_info, - inputs=[ - model_path - ], - outputs=[ - output_info - ], - api_name="read_model" - ) - refresh_model.click( - fn=lambda: change_models_choices()[0], - inputs=[], - outputs=[ - model_path - ] - ) \ No newline at end of file diff --git a/main/app/tabs/extra/child/settings.py b/main/app/tabs/extra/child/settings.py deleted file mode 100644 index 64b845734ed9f310bcd466410eb1100c935091a6..0000000000000000000000000000000000000000 --- a/main/app/tabs/extra/child/settings.py +++ /dev/null @@ -1,217 +0,0 @@ -import os -import sys - -import gradio as gr - -sys.path.append(os.getcwd()) - -from main.app.core.utils import stop_pid -from main.app.core.ui import change_fp, run_commands - -from main.app.core.restart import restart - -from main.app.variables import ( - font, - theme, - config, - configs, - language, - translations -) - -reload_js = """ -() => { - function Checking() { - fetch(window.location.origin) - .then(response => { - if (response.ok) { - location.reload(); - } else { - setTimeout(Checking, 3000); - } - }) - .catch(err => { - setTimeout(Checking, 3000); - }); - } - setTimeout(Checking, 10000); -} -""" - -def settings_tab(app): - with gr.Row(): - gr.Markdown(translations["settings_markdown_2"]) - with gr.Row(): - toggle_button = gr.Button( - translations["change_light_dark"], - variant="secondary", - scale=2 - ) - with gr.Row(): - with gr.Column(): - language_dropdown = gr.Dropdown( - label=translations["lang"], - interactive=True, - info=translations["lang_restart"], - choices=configs.get("support_language", "vi-VN"), - value=language - ) - with gr.Column(): - theme_dropdown = gr.Dropdown( - label=translations["theme"], - interactive=True, - info=translations["theme_restart"], - choices=configs.get("themes", theme), - value=theme, - allow_custom_value=True - ) - with gr.Column(): - font_choice = gr.Textbox( - label=translations["font"], - info=translations["font_info"], - value=font, - interactive=True - ) - with gr.Row(): - restart_button = gr.Button( - translations["restart_button"] - ) - with gr.Row(): - with gr.Column(): - with gr.Group(): - with gr.Row(): - bf16_checkbox = gr.Checkbox( - label=translations["bf16"], - value=False, - interactive=config.device.startswith(("cuda", "xpu")) and config.bf16_support - ) - tf32_checkbox = gr.Checkbox( - label=translations["tf32"], - value=False, - interactive=config.device.startswith("cuda") and config.tf32_support - ) - fp_choice = gr.Radio( - choices=["fp16","fp32"], - value="fp16" if configs.get("fp16", False) else "fp32", - label=translations["precision"], - info=translations["precision_info"], - interactive=config.allow_is_half - ) - fp_button = gr.Button( - translations["update_precision"], - variant="primary", - scale=2 - ) - with gr.Row(): - commands = gr.Textbox( - label=translations["commands"], - info=translations["commands_info"], - value="", - interactive=True - ) - with gr.Row(): - run_commands_button = gr.Button( - translations["run_commands"] - ) - with gr.Row(): - with gr.Column(): - with gr.Accordion(translations["stop"], open=False, visible=True): - separate_stop = gr.Button( - translations["stop_separate"] - ) - convert_stop = gr.Button( - translations["stop_convert"] - ) - create_dataset_stop = gr.Button( - translations["stop_create_dataset"] - ) - with gr.Accordion(translations["stop_training"], open=False): - model_name_stop = gr.Textbox( - label=translations["modelname"], - info=translations["training_model_name"], - value="", - placeholder=translations["modelname"], - interactive=True - ) - preprocess_stop = gr.Button( - translations["stop_preprocess"] - ) - extract_stop = gr.Button( - translations["stop_extract"] - ) - train_stop = gr.Button( - translations["stop_training"] - ) - with gr.Row(): - toggle_button.click( - fn=None, - js="() => {document.body.classList.toggle('dark')}" - ) - fp_button.click( - fn=change_fp, - inputs=[ - fp_choice, - bf16_checkbox, - tf32_checkbox - ], - outputs=[ - fp_choice - ] - ) - run_commands_button.click( - fn=run_commands, - inputs=[ - commands - ], - outputs=[] - ) - with gr.Row(): - restart_button.click( - fn=lambda lang, theme, font: restart(lang, theme, font, app), - inputs=[language_dropdown, theme_dropdown, font_choice], - outputs=[] - ) - restart_button.click( - fn=None, - js=reload_js, - inputs=[], - outputs=[] - ) - with gr.Row(): - separate_stop.click( - fn=lambda: stop_pid("separate_pid", None, False), - inputs=[], - outputs=[] - ) - convert_stop.click( - fn=lambda: stop_pid("convert_pid", None, False), - inputs=[], - outputs=[] - ) - create_dataset_stop.click( - fn=lambda: stop_pid("create_dataset_pid", None, False), - inputs=[], - outputs=[] - ) - with gr.Row(): - preprocess_stop.click( - fn=lambda model_name_stop: stop_pid("preprocess_pid", model_name_stop, False), - inputs=[ - model_name_stop - ], - outputs=[] - ) - extract_stop.click( - fn=lambda model_name_stop: stop_pid("extract_pid", model_name_stop, False), - inputs=[ - model_name_stop - ], - outputs=[] - ) - train_stop.click( - fn=lambda model_name_stop: stop_pid("train_pid", model_name_stop, True), - inputs=[ - model_name_stop - ], - outputs=[] - ) \ No newline at end of file diff --git a/main/app/tabs/extra/extra.py b/main/app/tabs/extra/extra.py deleted file mode 100644 index fd2b90542e3cf7df83795233f569c441b534482d..0000000000000000000000000000000000000000 --- a/main/app/tabs/extra/extra.py +++ /dev/null @@ -1,62 +0,0 @@ -import os -import sys - -import gradio as gr - -sys.path.append(os.getcwd()) - -from main.app.variables import ( - configs, - translations -) - -from main.app.tabs.extra.child.fushion import fushion_tab -from main.app.tabs.extra.child.analyzer import analyzer_tab -from main.app.tabs.extra.child.settings import settings_tab -from main.app.tabs.extra.child.read_model import read_model_tab -from main.app.tabs.extra.child.f0_extract import f0_extract_tab -from main.app.tabs.extra.child.create_srt import create_srt_tab -from main.app.tabs.extra.child.convert_model_svc import convert_svc_model_tab -from main.app.tabs.extra.child.convert_model_onnx import convert_onnx_model_tab - -def extra_tab(app): - with gr.TabItem(translations["extra"]): - if configs.get("fushion_tab", True): - with gr.TabItem(translations["fushion"]): - gr.Markdown(translations["fushion_markdown"]) - fushion_tab() - - if configs.get("read_tab", True): - with gr.TabItem(translations["read_model"]): - gr.Markdown(translations["read_model_markdown"]) - read_model_tab() - - if configs.get("onnx_tab", True): - with gr.TabItem(translations["convert_model"]): - gr.Markdown(translations["pytorch2onnx"]) - convert_onnx_model_tab() - - if configs.get("svc_tab", True): - with gr.TabItem(translations["convert_model_svc"]): - gr.Markdown(translations["convert_model_svc_markdown"]) - convert_svc_model_tab() - - if configs.get("f0_extractor_tab", True): - with gr.TabItem(translations["f0_extractor_tab"]): - gr.Markdown(translations["f0_extractor_markdown"]) - f0_extract_tab() - - if configs.get("create_srt_tab", True): - with gr.TabItem(translations["create_srt_tab"]): - gr.Markdown(translations["create_srt_markdown"]) - create_srt_tab() - - if configs.get("analyzer_tab", True): - with gr.TabItem(translations["analyzer_tab"]): - gr.Markdown(translations["analyzer_markdown"]) - analyzer_tab() - - if configs.get("settings_tab", True): - with gr.TabItem(translations["settings"]): - gr.Markdown(translations["settings_markdown"]) - settings_tab(app) \ No newline at end of file diff --git a/main/app/tabs/inference/child/convert.py b/main/app/tabs/inference/child/convert.py deleted file mode 100644 index 0f9163e74a89675d80d952a1b6e2d02b5a12a065..0000000000000000000000000000000000000000 --- a/main/app/tabs/inference/child/convert.py +++ /dev/null @@ -1,1142 +0,0 @@ -import os -import sys - -import gradio as gr - -sys.path.append(os.getcwd()) - -from main.app.core.presets import ( - load_presets, - save_presets -) - -from main.app.core.inference import ( - convert_audio, - convert_selection -) - -from main.app.variables import ( - configs, - f0_file, - method_f0, - file_types, - model_name, - index_path, - audio_params, - presets_file, - translations, - embedders_mode, - embedders_model, - paths_for_files, - sample_rate_choice, - hybrid_f0_method, - export_format_choices, -) - -from main.app.core.ui import ( - visible, - unlock_f0, - get_index, - shutil_move, - hoplength_show, - change_f0_choices, - index_strength_show, - change_embedders_mode, - change_models_choices, - change_preset_choices, - change_audios_choices, - valueFalse_interactive, - change_backing_choices, - get_speakers_id_and_architecture -) - -def convert_tab(): - with gr.Row(): - gr.Markdown(translations["convert_info"]) - with gr.Row(): - with gr.Column(): - with gr.Group(): - with gr.Row(): - cleaner = gr.Checkbox( - label=translations["clear_audio"], - value=False, - interactive=True - ) - autotune = gr.Checkbox( - label=translations["autotune"], - value=False, - interactive=True - ) - use_audio = gr.Checkbox( - label=translations["use_audio"], - value=False, - interactive=True - ) - checkpointing = gr.Checkbox( - label=translations["memory_efficient_training"], - value=False, - interactive=True - ) - with gr.Row(): - use_original = gr.Checkbox( - label=translations["convert_original"], - value=False, - interactive=True, - visible=False - ) - convert_backing = gr.Checkbox( - label=translations["convert_backing"], - value=False, - interactive=True, - visible=False - ) - not_merge_backing = gr.Checkbox( - label=translations["not_merge_backing"], - value=False, - interactive=True, - visible=False - ) - merge_instrument = gr.Checkbox( - label=translations["merge_instruments"], - value=False, - interactive=True, - visible=False - ) - with gr.Row(): - pitch = gr.Slider( - minimum=-24, - maximum=24, - step=1, - info=translations["pitch_info"], - label=translations["pitch"], - value=0, - interactive=True - ) - clean_strength = gr.Slider( - label=translations["clean_strength"], - info=translations["clean_strength_info"], - minimum=0, - maximum=1, - value=0.5, - step=0.1, - interactive=True, - visible=False - ) - with gr.Row(): - with gr.Column(): - convert_select_audio = gr.Dropdown( - label=translations["select_separate"], - choices=[], - value="", - interactive=True, - allow_custom_value=True, - visible=False - ) - convert_select_audio_button = gr.Button( - translations["convert_audio"], - visible=False - ) - with gr.Row(): - with gr.Column(): - convert_audio_button = gr.Button( - translations["convert_audio"], - variant="primary" - ) - with gr.Row(): - with gr.Column(): - drop_audio_file = gr.Files( - label=translations["drop_audio"], - file_types=file_types - ) - play_audio = gr.Audio( - interactive=False, - label=translations["input_audio"], - **audio_params - ) - with gr.Column(): - with gr.Accordion( - translations["model_accordion"], - open=True - ): - with gr.Row(): - model_pth = gr.Dropdown( - label=translations["model_name"], - choices=model_name, - value=model_name[0] if len(model_name) >= 1 else "", - interactive=True, - allow_custom_value=True - ) - model_index = gr.Dropdown( - label=translations["index_path"], - choices=index_path, - value=index_path[0] if len(index_path) >= 1 else "", - interactive=True, - allow_custom_value=True - ) - with gr.Row(): - refresh_model = gr.Button( - translations["refresh"] - ) - with gr.Row(): - index_strength = gr.Slider( - label=translations["index_strength"], - info=translations["index_strength_info"], - minimum=0, - maximum=1, - value=0.5, - step=0.01, - interactive=True, - visible=True - ) - nprobe = gr.Slider( - label=translations["nprobe"], - info=translations["nprobe_info"], - minimum=1, - maximum=64, - value=9, - step=1, - interactive=True, - visible=True - ) - with gr.Row(): - sid_dict, architecture_dict = get_speakers_id_and_architecture(model_pth.value) - if sid_dict["visible"] == "hidden": sid_dict["visible"] = False - if architecture_dict["visible"] == "hidden": architecture_dict["visible"] = False - - sids = gr.Dropdown( - label=translations["sids_label"], - info=translations["sids_info"], - choices=sid_dict["choices"], - value=sid_dict["value"], - visible=sid_dict["visible"], - interactive=True - ) - with gr.Accordion( - translations["input_output"], - open=False - ): - with gr.Column(): - export_format = gr.Radio( - label=translations["export_format"], - info=translations["export_info"], - choices=export_format_choices, - value="wav", - interactive=True - ) - input_audio = gr.Dropdown( - label=translations["audio_path"], - value="", - choices=paths_for_files, - info=translations["provide_audio"], - allow_custom_value=True, - interactive=True - ) - output_audio = gr.Textbox( - label=translations["output_path"], - value="audios/output.wav", - placeholder="audios/output.wav", - info=translations["output_path_info"], - interactive=True - ) - with gr.Column(): - refresh_audio = gr.Button( - translations["refresh"] - ) - with gr.Accordion( - translations["setting"], - open=False - ): - with gr.Accordion( - translations["f0_method"], - open=False - ): - with gr.Group(): - with gr.Row(): - predictor_onnx = gr.Checkbox( - label=translations["predictor_onnx"], - info=translations["predictor_onnx_info"], - value=False, - interactive=True - ) - unlock_full_method = gr.Checkbox( - label=translations["f0_unlock"], - info=translations["f0_unlock_info"], - value=False, - interactive=True - ) - f0_method = gr.Radio( - label=translations["f0_method"], - info=translations["f0_method_info"], - choices=method_f0, - value="rmvpe", - interactive=True - ) - hybrid_method = gr.Dropdown( - label=translations["f0_method_hybrid"], - info=translations["f0_method_hybrid_info"], - choices=hybrid_f0_method, - value=hybrid_f0_method[0], - interactive=True, - allow_custom_value=True, - visible=False - ) - hop_length = gr.Slider( - label=translations['hop_length'], - info=translations["hop_length_info"], - minimum=64, - maximum=512, - value=160, - step=1, - interactive=True, - visible=False - ) - alpha = gr.Slider( - label=translations["alpha_label"], - info=translations["alpha_info"], - minimum=0.1, - maximum=1, - value=0.5, - step=0.1, - interactive=True, - visible=False - ) - with gr.Accordion( - translations["f0_file"], - open=False - ): - upload_f0_file = gr.File( - label=translations["upload_f0"], - file_types=[".txt"] - ) - f0_file_dropdown = gr.Dropdown( - label=translations["f0_file_2"], - value="", - choices=f0_file, - allow_custom_value=True, - interactive=True - ) - refresh_f0_file = gr.Button( - translations["refresh"] - ) - with gr.Accordion( - translations["hubert_model"], - open=False - ): - embedders_mix = gr.Checkbox( - label=translations["embedders_mix"], - info=translations["embedders_mix_info"], - value=False, - interactive=True - ) - embedder_mode = gr.Radio( - label=translations["embed_mode"], - info=translations["embed_mode_info"], - value="fairseq", - choices=embedders_mode, - interactive=True, - visible=True - ) - embedders = gr.Radio( - label=translations["hubert_model"], - info=translations["hubert_info"], - choices=embedders_model, - value="hubert_base", - interactive=True - ) - custom_embedders = gr.Textbox( - label=translations["modelname"], - info=translations["modelname_info"], - value="", - placeholder="hubert_base", - interactive=True, - visible=False - ) - with gr.Column(visible=False) as embedders_mix_column: - embedders_mix_layers = gr.Slider( - label=translations["embedders_mix_layers"], - info=translations["embedders_mix_layers_info"], - minimum=1, - maximum=12, - value=9, - step=1, - interactive=True - ) - embedders_mix_ratio = gr.Slider( - label=translations["embedders_mix_ratio"], - info=translations["embedders_mix_ratio_info"], - minimum=0.1, - maximum=1, - value=0.5, - step=0.1, - interactive=True - ) - with gr.Accordion( - translations["use_presets"], - open=False - ): - with gr.Row(): - presets_name = gr.Dropdown( - label=translations["file_preset"], - choices=presets_file, - value=presets_file[0] if len(presets_file) > 0 else '', - interactive=True, - allow_custom_value=True - ) - with gr.Row(): - load_preset_button = gr.Button( - translations["load_file"], - variant="primary" - ) - refresh_preset_button = gr.Button( - translations["refresh"] - ) - with gr.Accordion( - translations["export_file"], - open=False - ): - with gr.Row(): - with gr.Column(): - with gr.Group(): - with gr.Row(): - cleaner_checkbox = gr.Checkbox( - label=translations["save_clean"], - value=True, - interactive=True - ) - autotune_checkbox = gr.Checkbox( - label=translations["save_autotune"], - value=True, - interactive=True - ) - pitch_checkbox = gr.Checkbox( - label=translations["save_pitch"], - value=True, - interactive=True - ) - index_strength_checkbox = gr.Checkbox( - label=translations["save_index_2"], - value=True, - interactive=True - ) - resample_sr_checkbox = gr.Checkbox( - label=translations["save_resample"], - value=True, - interactive=True - ) - filter_radius_checkbox = gr.Checkbox( - label=translations["save_filter"], - value=True, - interactive=True - ) - rms_mix_rate_checkbox = gr.Checkbox( - label=translations["save_envelope"], - value=True, - interactive=True - ) - protect_checkbox = gr.Checkbox( - label=translations["save_protect"], - value=True, - interactive=True - ) - split_audio_checkbox = gr.Checkbox( - label=translations["save_split"], - value=True, - interactive=True - ) - formant_shifting_checkbox = gr.Checkbox( - label=translations["formantshift"], - value=True, - interactive=True - ) - with gr.Row(): - with gr.Column(): - name_to_save_file = gr.Textbox( - label=translations["filename_to_save"] - ) - save_file_button = gr.Button( - translations["export_file"] - ) - with gr.Row(): - upload_presets = gr.Files( - label=translations["upload_presets"], - file_types=[".conversion.json"] - ) - with gr.Column(): - with gr.Group(): - with gr.Row(): - split_audio = gr.Checkbox( - label=translations["split_audio"], - value=False, - interactive=True - ) - formant_shifting = gr.Checkbox( - label=translations["formantshift"], - value=False, - interactive=True - ) - with gr.Row(): - proposal_pitch = gr.Checkbox( - label=translations["proposal_pitch"], - value=False, - interactive=True - ) - audio_processing = gr.Checkbox( - label=translations["audio_processing"], - value=False, - interactive=True - ) - with gr.Row(): - audio_upscaler = gr.Checkbox( - label=translations["audio_upscaler"], - info=translations["audio_upscaler_info"], - value=False, - interactive=True - ) - resample_sr = gr.Radio( - choices=[0]+sample_rate_choice, - label=translations["resample"], - info=translations["resample_info"], - value=0, - interactive=True - ) - proposal_pitch_threshold = gr.Slider( - minimum=50.0, - maximum=1200.0, - label=translations["proposal_pitch_threshold"], - info=translations["proposal_pitch_threshold_info"], - value=255.0, - step=0.1, - interactive=True, - visible=False - ) - f0_autotune_strength = gr.Slider( - minimum=0, - maximum=1, - label=translations["autotune_rate"], - info=translations["autotune_rate_info"], - value=1, - step=0.1, - interactive=True, - visible=False - ) - filter_radius = gr.Slider( - minimum=0, - maximum=7, - label=translations["filter_radius"], - info=translations["filter_radius_info"], - value=3, - step=1, - interactive=True - ) - rms_mix_rate = gr.Slider( - minimum=0, - maximum=1, - label=translations["rms_mix_rate"], - info=translations["rms_mix_rate_info"], - value=1, - step=0.1, - interactive=True - ) - protect = gr.Slider( - minimum=0, - maximum=1, - label=translations["protect"], - info=translations["protect_info"], - value=0.5, - step=0.01, - interactive=True - ) - noise_scale = gr.Slider( - minimum=0.1, - maximum=1.0, - label=translations["noise_scale"], - info=translations["noise_scale_info"], - value=0.4, - step=0.01, - interactive=True, - visible=architecture_dict["visible"] - ) - with gr.Row(): - formant_qfrency = gr.Slider( - value=1.0, - label=translations["formant_qfrency"], - info=translations["formant_qfrency"], - minimum=0.0, - maximum=16.0, - step=0.1, - interactive=True, - visible=False - ) - formant_timbre = gr.Slider( - value=1.0, - label=translations["formant_timbre"], - info=translations["formant_timbre"], - minimum=0.0, - maximum=16.0, - step=0.1, - interactive=True, - visible=False - ) - with gr.Row(): - gr.Markdown(translations["output_convert"]) - with gr.Row(): - main_convert = gr.Audio( - interactive=False, - label=translations["main_convert"], - visible=True, - **audio_params - ) - backing_convert = gr.Audio( - interactive=False, - label=translations["convert_backing"], - visible=False, - **audio_params - ) - main_backing = gr.Audio( - interactive=False, - label=translations["main_or_backing"], - visible=False, - **audio_params - ) - with gr.Row(): - original_convert = gr.Audio( - interactive=False, - label=translations["convert_original"], - visible=False, - **audio_params - ) - vocal_instrument = gr.Audio( - interactive=False, - label=translations["voice_or_instruments"], - visible=False, - **audio_params - ) - with gr.Row(): - upload_f0_file.upload( - fn=lambda inp: shutil_move(inp.name, configs["f0_path"]), - inputs=[ - upload_f0_file - ], - outputs=[ - f0_file_dropdown - ] - ) - refresh_f0_file.click( - fn=change_f0_choices, - inputs=[], - outputs=[ - f0_file_dropdown - ] - ) - unlock_full_method.change( - fn=unlock_f0, - inputs=[ - unlock_full_method - ], - outputs=[ - f0_method - ] - ) - with gr.Row(): - load_preset_button.click( - fn=load_presets, - inputs=[ - presets_name, - cleaner, - autotune, - pitch, - clean_strength, - index_strength, - resample_sr, - filter_radius, - rms_mix_rate, - protect, - split_audio, - f0_autotune_strength, - formant_shifting, - formant_qfrency, - formant_timbre, - proposal_pitch, - proposal_pitch_threshold, - embedders_mix, - embedders_mix_layers, - embedders_mix_ratio - ], - outputs=[ - cleaner, - autotune, - pitch, - clean_strength, - index_strength, - resample_sr, - filter_radius, - rms_mix_rate, - protect, - split_audio, - f0_autotune_strength, - formant_shifting, - formant_qfrency, - formant_timbre, - proposal_pitch, - proposal_pitch_threshold, - embedders_mix, - embedders_mix_layers, - embedders_mix_ratio - ] - ) - refresh_preset_button.click( - fn=change_preset_choices, - inputs=[], - outputs=[ - presets_name - ] - ) - save_file_button.click( - fn=save_presets, - inputs=[ - name_to_save_file, - cleaner, - autotune, - pitch, - clean_strength, - index_strength, - resample_sr, - filter_radius, - rms_mix_rate, - protect, - split_audio, - f0_autotune_strength, - cleaner_checkbox, - autotune_checkbox, - pitch_checkbox, - index_strength_checkbox, - resample_sr_checkbox, - filter_radius_checkbox, - rms_mix_rate_checkbox, - protect_checkbox, - split_audio_checkbox, - formant_shifting_checkbox, - formant_shifting, - formant_qfrency, - formant_timbre, - proposal_pitch, - proposal_pitch_threshold, - embedders_mix, - embedders_mix_layers, - embedders_mix_ratio - ], - outputs=[ - presets_name - ] - ) - with gr.Row(): - upload_presets.upload( - fn=lambda presets_in: [ - shutil_move(preset.name, configs["presets_path"]) - for preset in presets_in - ][0], - inputs=[ - upload_presets - ], - outputs=[ - presets_name - ] - ) - autotune.change( - fn=visible, - inputs=[ - autotune - ], - outputs=[ - f0_autotune_strength - ] - ) - use_audio.change( - fn=lambda a: [ - visible(a), - visible(a), - visible(a), - visible(a), - visible(a), - valueFalse_interactive(a), - valueFalse_interactive(a), - valueFalse_interactive(a), - valueFalse_interactive(a), - visible(not a), - visible(not a), - visible(not a), - visible(not a) - ], - inputs=[ - use_audio - ], - outputs=[ - main_backing, - use_original, - convert_backing, - not_merge_backing, - merge_instrument, - use_original, - convert_backing, - not_merge_backing, - merge_instrument, - input_audio, - output_audio, - drop_audio_file, - play_audio - ] - ) - with gr.Row(): - convert_backing.change( - fn=lambda a,b: [ - change_backing_choices(a, b), - visible(a) - ], - inputs=[ - convert_backing, - not_merge_backing - ], - outputs=[ - use_original, - backing_convert - ] - ) - use_original.change( - fn=lambda audio, original: [ - visible(original), - visible(not original), - visible(audio and not original), - valueFalse_interactive(not original), - valueFalse_interactive(not original) - ], - inputs=[ - use_audio, - use_original - ], - outputs=[ - original_convert, - main_convert, - main_backing, - convert_backing, - not_merge_backing - ] - ) - cleaner.change( - fn=visible, - inputs=[ - cleaner - ], - outputs=[ - clean_strength - ] - ) - with gr.Row(): - merge_instrument.change( - fn=visible, - inputs=[ - merge_instrument - ], - outputs=[ - vocal_instrument - ] - ) - not_merge_backing.change( - fn=lambda audio, merge, cvb: [ - visible(audio and not merge), - change_backing_choices(cvb, merge) - ], - inputs=[ - use_audio, - not_merge_backing, - convert_backing - ], - outputs=[ - main_backing, - use_original - ] - ) - f0_method.change( - fn=lambda f0_method, hybrid: [ - visible(f0_method == "hybrid"), - visible(f0_method == "hybrid"), - hoplength_show(f0_method, hybrid) - ], - inputs=[ - f0_method, - hybrid_method - ], - outputs=[ - hybrid_method, - alpha, - hop_length - ] - ) - with gr.Row(): - hybrid_method.change( - fn=hoplength_show, - inputs=[ - f0_method, - hybrid_method - ], - outputs=[ - hop_length - ] - ) - refresh_model.click( - fn=change_models_choices, - inputs=[], - outputs=[ - model_pth, - model_index - ] - ) - model_pth.change( - fn=get_index, - inputs=[ - model_pth - ], - outputs=[ - model_index - ] - ) - with gr.Row(): - drop_audio_file.upload( - fn=lambda audio_in: [ - shutil_move(audio.name, configs["audios_path"]) - for audio in audio_in - ][0], - inputs=[ - drop_audio_file - ], - outputs=[ - input_audio - ] - ) - input_audio.change( - fn=lambda audio: audio if os.path.isfile(audio) else None, - inputs=[ - input_audio - ], - outputs=[ - play_audio - ] - ) - formant_shifting.change( - fn=lambda a: [ - visible(a) - for _ in range(2) - ], - inputs=[ - formant_shifting - ], - outputs=[ - formant_qfrency, - formant_timbre - ] - ) - with gr.Row(): - embedders.change( - fn=lambda embedders: visible(embedders == "custom"), - inputs=[ - embedders - ], - outputs=[ - custom_embedders - ] - ) - refresh_audio.click( - fn=change_audios_choices, - inputs=[ - input_audio - ], - outputs=[ - input_audio - ] - ) - model_index.change( - fn=index_strength_show, - inputs=[ - model_index - ], - outputs=[ - index_strength, - nprobe - ] - ) - with gr.Row(): - convert_audio_button.click( - fn=lambda: visible(False), - inputs=[], - outputs=[ - convert_audio_button - ] - ) - convert_select_audio_button.click( - fn=lambda: [ - visible(False) - for _ in range(2) - ], - inputs=[], - outputs=[ - convert_select_audio, - convert_select_audio_button - ] - ) - with gr.Row(): - proposal_pitch.change( - fn=visible, - inputs=[ - proposal_pitch - ], - outputs=[ - proposal_pitch_threshold - ] - ) - embedder_mode.change( - fn=change_embedders_mode, - inputs=[ - embedder_mode - ], - outputs=[ - embedders - ] - ) - model_pth.change( - fn=get_speakers_id_and_architecture, - inputs=[ - model_pth - ], - outputs=[ - sids, - noise_scale - ] - ) - with gr.Row(): - embedders_mix.change( - fn=visible, - inputs=[ - embedders_mix - ], - outputs=[ - embedders_mix_column - ] - ) - convert_audio_button.click( - fn=convert_selection, - inputs=[ - cleaner, - autotune, - use_audio, - use_original, - convert_backing, - not_merge_backing, - merge_instrument, - pitch, - clean_strength, - model_pth, - model_index, - index_strength, - input_audio, - output_audio, - export_format, - f0_method, - hybrid_method, - hop_length, - embedders, - custom_embedders, - resample_sr, - filter_radius, - rms_mix_rate, - protect, - split_audio, - f0_autotune_strength, - checkpointing, - predictor_onnx, - formant_shifting, - formant_qfrency, - formant_timbre, - f0_file_dropdown, - embedder_mode, - proposal_pitch, - proposal_pitch_threshold, - audio_processing, - alpha, - sids, - embedders_mix, - embedders_mix_layers, - embedders_mix_ratio, - noise_scale, - nprobe, - audio_upscaler - ], - outputs=[ - convert_select_audio, - main_convert, - backing_convert, - main_backing, - original_convert, - vocal_instrument, - convert_audio_button, - convert_select_audio_button - ], - api_name="convert_selection" - ) - convert_select_audio_button.click( - fn=convert_audio, - inputs=[ - cleaner, - autotune, - use_audio, - use_original, - convert_backing, - not_merge_backing, - merge_instrument, - pitch, - clean_strength, - model_pth, - model_index, - index_strength, - input_audio, - output_audio, - export_format, - f0_method, - hybrid_method, - hop_length, - embedders, - custom_embedders, - resample_sr, - filter_radius, - rms_mix_rate, - protect, - split_audio, - f0_autotune_strength, - convert_select_audio, - checkpointing, - predictor_onnx, - formant_shifting, - formant_qfrency, - formant_timbre, - f0_file_dropdown, - embedder_mode, - proposal_pitch, - proposal_pitch_threshold, - audio_processing, - alpha, - sids, - embedders_mix, - embedders_mix_layers, - embedders_mix_ratio, - noise_scale, - nprobe, - audio_upscaler - ], - outputs=[ - main_convert, - backing_convert, - main_backing, - original_convert, - vocal_instrument, - convert_audio_button - ], - api_name="convert_audio" - ) \ No newline at end of file diff --git a/main/app/tabs/inference/child/convert_tts.py b/main/app/tabs/inference/child/convert_tts.py deleted file mode 100644 index c9fafc7ce527859aa4fe95cb26c42c917a5678bd..0000000000000000000000000000000000000000 --- a/main/app/tabs/inference/child/convert_tts.py +++ /dev/null @@ -1,979 +0,0 @@ -import os -import sys - -import gradio as gr - -sys.path.append(os.getcwd()) - -from main.app.core.tts import TTS -from main.app.core.process import process_input -from main.app.core.inference import convert_tts -from main.app.core.utils import google_translate - -from main.app.core.presets import ( - save_presets, - load_presets -) - -from main.app.variables import ( - configs, - edgetts, - f0_file, - method_f0, - model_name, - index_path, - presets_file, - translations, - audio_params, - embedders_mode, - embedders_model, - google_tts_voice, - hybrid_f0_method, - sample_rate_choice, - export_format_choices -) - -from main.app.core.ui import ( - visible, - unlock_f0, - get_index, - shutil_move, - hoplength_show, - change_f0_choices, - index_strength_show, - change_models_choices, - change_preset_choices, - change_embedders_mode, - change_tts_voice_choices, - get_speakers_id_and_architecture -) - -def convert_tts_tab(): - with gr.Row(): - gr.Markdown(translations["convert_text_markdown_2"]) - with gr.Row(): - with gr.Column(): - with gr.Group(): - with gr.Row(): - use_txt = gr.Checkbox( - label=translations["input_txt"], - value=False, - interactive=True - ) - google_tts_check_box = gr.Checkbox( - label=translations["googletts"], - value=False, - interactive=True - ) - prompt = gr.Textbox( - label=translations["text_to_speech"], - value="", - placeholder="Hello Words", - lines=5, - max_lines=5 - ) - with gr.Column(): - speed = gr.Slider( - label=translations["voice_speed"], - info=translations["voice_speed_info"], - minimum=-100, - maximum=100, - value=0, - step=1 - ) - pitch = gr.Slider( - minimum=-24, - maximum=24, - step=1, - info=translations["pitch_info"], - label=translations["pitch"], - value=0, - interactive=True - ) - with gr.Row(): - tts_button = gr.Button( - translations["tts_1"], - variant="primary", - scale=2 - ) - convert_button = gr.Button( - translations["tts_2"], - variant="secondary", - scale=2 - ) - with gr.Row(): - with gr.Column(): - txt_input = gr.File( - label=translations["drop_text"], - file_types=[".txt", ".srt", ".docx"], - visible=False - ) - tts_voice = gr.Dropdown( - label=translations["voice"], - choices=edgetts, - interactive=True, - value="vi-VN-NamMinhNeural" - ) - tts_pitch = gr.Slider( - minimum=-24, - maximum=24, - step=1, - info=translations["pitch_info_2"], - label=translations["pitch"], - value=0, - interactive=True - ) - with gr.Accordion( - translations["translate"], - open=False - ): - with gr.Row(): - source_lang = gr.Dropdown( - label=translations["source_lang"], - choices=["auto"]+google_tts_voice, - interactive=True, - value="auto" - ) - target_lang = gr.Dropdown( - label=translations["target_lang"], - choices=google_tts_voice, - interactive=True, - value="en" - ) - translate_button = gr.Button( - translations["translate"] - ) - with gr.Column(): - with gr.Accordion( - translations["model_accordion"], - open=True - ): - with gr.Row(): - model_pth = gr.Dropdown( - label=translations["model_name"], - choices=model_name, - value=model_name[0] if len(model_name) >= 1 else "", - interactive=True, - allow_custom_value=True - ) - model_index = gr.Dropdown( - label=translations["index_path"], - choices=index_path, - value=index_path[0] if len(index_path) >= 1 else "", - interactive=True, - allow_custom_value=True - ) - with gr.Row(): - refresh = gr.Button( - translations["refresh"] - ) - with gr.Row(): - index_strength = gr.Slider( - label=translations["index_strength"], - info=translations["index_strength_info"], - minimum=0, - maximum=1, - value=0.5, - step=0.01, - interactive=True, - visible=True - ) - nprobe = gr.Slider( - label=translations["nprobe"], - info=translations["nprobe_info"], - minimum=1, - maximum=64, - value=9, - step=1, - interactive=True, - visible=True - ) - with gr.Row(): - sid_dict, architecture_dict = get_speakers_id_and_architecture(model_pth.value) - if sid_dict["visible"] == "hidden": sid_dict["visible"] = False - if architecture_dict["visible"] == "hidden": architecture_dict["visible"] = False - - sids = gr.Dropdown( - label=translations["sids_label"], - info=translations["sids_info"], - choices=sid_dict["choices"], - value=sid_dict["value"], - visible=sid_dict["visible"], - interactive=True - ) - with gr.Accordion( - translations["output_path"], - open=False - ): - export_format = gr.Radio( - label=translations["export_format"], - info=translations["export_info"], - choices=export_format_choices, - value="wav", - interactive=True - ) - output_audio_tts = gr.Textbox( - label=translations["output_tts"], - value="audios/tts.wav", - placeholder="audios/tts.wav", - info=translations["tts_output"], - interactive=True - ) - output_audio = gr.Textbox( - label=translations["output_tts_convert"], - value="audios/tts-convert.wav", - placeholder="audios/tts-convert.wav", - info=translations["tts_output"], - interactive=True - ) - with gr.Accordion( - translations["setting"], - open=False - ): - with gr.Accordion( - translations["f0_method"], - open=False - ): - with gr.Group(): - with gr.Row(): - predictor_onnx = gr.Checkbox( - label=translations["predictor_onnx"], - info=translations["predictor_onnx_info"], - value=False, - interactive=True - ) - unlock_full_method = gr.Checkbox( - label=translations["f0_unlock"], - info=translations["f0_unlock_info"], - value=False, - interactive=True - ) - f0_method = gr.Radio( - label=translations["f0_method"], - info=translations["f0_method_info"], - choices=method_f0, - value="rmvpe", - interactive=True - ) - hybrid_f0method = gr.Dropdown( - label=translations["f0_method_hybrid"], - info=translations["f0_method_hybrid_info"], - choices=hybrid_f0_method, - value=hybrid_f0_method[0], - interactive=True, - allow_custom_value=True, - visible=False - ) - hop_length = gr.Slider( - label=translations['hop_length'], - info=translations["hop_length_info"], - minimum=64, - maximum=512, - value=160, - step=1, - interactive=True, - visible=False - ) - alpha = gr.Slider( - label=translations["alpha_label"], - info=translations["alpha_info"], - minimum=0.1, - maximum=1, - value=0.5, - step=0.1, - interactive=True, - visible=False - ) - with gr.Accordion( - translations["f0_file"], - open=False - ): - upload_f0_file = gr.File( - label=translations["upload_f0"], - file_types=[".txt"] - ) - f0_file_dropdown = gr.Dropdown( - label=translations["f0_file_2"], - value="", - choices=f0_file, - allow_custom_value=True, - interactive=True - ) - refresh_f0_file = gr.Button( - translations["refresh"] - ) - with gr.Accordion( - translations["hubert_model"], - open=False - ): - embedders_mix = gr.Checkbox( - label=translations["embedders_mix"], - info=translations["embedders_mix_info"], - value=False, - interactive=True - ) - embedder_mode = gr.Radio( - label=translations["embed_mode"], - info=translations["embed_mode_info"], - value="fairseq", - choices=embedders_mode, - interactive=True, - visible=True - ) - embedders = gr.Radio( - label=translations["hubert_model"], - info=translations["hubert_info"], - choices=embedders_model, - value="hubert_base", - interactive=True - ) - custom_embedders = gr.Textbox( - label=translations["modelname"], - info=translations["modelname_info"], - value="", - placeholder="hubert_base", - interactive=True, - visible=False - ) - with gr.Column(visible=False) as embedders_mix_column: - embedders_mix_layers = gr.Slider( - label=translations["embedders_mix_layers"], - info=translations["embedders_mix_layers_info"], - minimum=1, - maximum=12, - value=9, - step=1, - interactive=True - ) - embedders_mix_ratio = gr.Slider( - label=translations["embedders_mix_ratio"], - info=translations["embedders_mix_ratio_info"], - minimum=0.1, - maximum=1, - value=0.5, - step=0.1, - interactive=True - ) - with gr.Accordion( - translations["use_presets"], - open=False - ): - with gr.Row(): - presets_name = gr.Dropdown( - label=translations["file_preset"], - choices=presets_file, - value=presets_file[0] if len(presets_file) > 0 else '', - interactive=True, - allow_custom_value=True - ) - with gr.Row(): - load_preset_button = gr.Button( - translations["load_file"], - variant="primary" - ) - refresh_preset_button = gr.Button( - translations["refresh"] - ) - with gr.Accordion( - translations["export_file"], - open=False - ): - with gr.Row(): - with gr.Column(): - with gr.Group(): - with gr.Row(): - cleaner_checkbox = gr.Checkbox( - label=translations["save_clean"], - value=True, - interactive=True - ) - autotune_checkbox = gr.Checkbox( - label=translations["save_autotune"], - value=True, - interactive=True - ) - pitch_checkbox = gr.Checkbox( - label=translations["save_pitch"], - value=True, - interactive=True - ) - index_strength_checkbox = gr.Checkbox( - label=translations["save_index_2"], - value=True, - interactive=True - ) - resample_sr_checkbox = gr.Checkbox( - label=translations["save_resample"], - value=True, - interactive=True - ) - filter_radius_checkbox = gr.Checkbox( - label=translations["save_filter"], - value=True, - interactive=True - ) - rms_mix_rate_checkbox = gr.Checkbox( - label=translations["save_envelope"], - value=True, - interactive=True - ) - protect_checkbox = gr.Checkbox( - label=translations["save_protect"], - value=True, - interactive=True - ) - split_audio_checkbox = gr.Checkbox( - label=translations["save_split"], - value=True, - interactive=True - ) - formant_shifting_checkbox = gr.Checkbox( - label=translations["formantshift"], - value=True, - interactive=True - ) - with gr.Row(): - with gr.Column(): - name_to_save_file = gr.Textbox( - label=translations["filename_to_save"] - ) - save_file_button = gr.Button( - translations["export_file"] - ) - with gr.Row(): - upload_presets = gr.Files( - label=translations["upload_presets"], - file_types=[".conversion.json"] - ) - with gr.Group(): - with gr.Row(): - audio_processing = gr.Checkbox( - label=translations["audio_processing"], - value=False, - interactive=True - ) - with gr.Row(): - formant_shifting = gr.Checkbox( - label=translations["formantshift"], - value=False, - interactive=True - ) - split_audio = gr.Checkbox( - label=translations["split_audio"], - value=False, - interactive=True - ) - cleaner = gr.Checkbox( - label=translations["clear_audio"], - value=False, - interactive=True - ) - with gr.Row(): - autotune = gr.Checkbox( - label=translations["autotune"], - value=False, - interactive=True - ) - checkpointing = gr.Checkbox( - label=translations["memory_efficient_training"], - value=False, - interactive=True - ) - proposal_pitch = gr.Checkbox( - label=translations["proposal_pitch"], - value=False, - interactive=True - ) - with gr.Row(): - audio_upscaler = gr.Checkbox( - label=translations["audio_upscaler"], - info=translations["audio_upscaler_info"], - value=False, - interactive=True - ) - with gr.Column(): - resample_sr = gr.Radio( - choices=[0]+sample_rate_choice, - label=translations["resample"], - info=translations["resample_info"], - value=0, - interactive=True - ) - proposal_pitch_threshold = gr.Slider( - minimum=50.0, - maximum=1200.0, - label=translations["proposal_pitch_threshold"], - info=translations["proposal_pitch_threshold_info"], - value=255.0, - step=0.1, - interactive=True, - visible=False - ) - f0_autotune_strength = gr.Slider( - minimum=0, - maximum=1, - label=translations["autotune_rate"], - info=translations["autotune_rate_info"], - value=1, - step=0.1, - interactive=True, - visible=False - ) - clean_strength = gr.Slider( - minimum=0, - maximum=1, - label=translations["clean_strength"], - info=translations["clean_strength_info"], - value=0.5, - step=0.1, - interactive=True, - visible=False - ) - filter_radius = gr.Slider( - minimum=0, - maximum=7, - label=translations["filter_radius"], - info=translations["filter_radius_info"], - value=3, - step=1, - interactive=True - ) - rms_mix_rate = gr.Slider( - minimum=0, - maximum=1, - label=translations["rms_mix_rate"], - info=translations["rms_mix_rate_info"], - value=1, - step=0.1, - interactive=True - ) - protect = gr.Slider( - minimum=0, - maximum=1, - label=translations["protect"], - info=translations["protect_info"], - value=0.5, - step=0.01, - interactive=True - ) - noise_scale = gr.Slider( - minimum=0.1, - maximum=1.0, - label=translations["noise_scale"], - info=translations["noise_scale_info"], - value=0.4, - step=0.01, - interactive=True, - visible=architecture_dict["visible"] - ) - with gr.Row(): - formant_qfrency = gr.Slider( - value=1.0, - label=translations["formant_qfrency"], - info=translations["formant_qfrency"], - minimum=0.0, - maximum=16.0, - step=0.1, - interactive=True, - visible=False - ) - formant_timbre = gr.Slider( - value=1.0, - label=translations["formant_timbre"], - info=translations["formant_timbre"], - minimum=0.0, - maximum=16.0, - step=0.1, - interactive=True, - visible=False - ) - with gr.Row(): - gr.Markdown(translations["output_tts_markdown"]) - with gr.Row(): - tts_voice_audio = gr.Audio( - interactive=False, - label=translations["output_text_to_speech"], - **audio_params - ) - tts_voice_convert = gr.Audio( - interactive=False, - label=translations["output_file_tts_convert"], - **audio_params - ) - with gr.Row(): - load_preset_button.click( - fn=load_presets, - inputs=[ - presets_name, - cleaner, - autotune, - pitch, - clean_strength, - index_strength, - resample_sr, - filter_radius, - rms_mix_rate, - protect, - split_audio, - f0_autotune_strength, - formant_shifting, - formant_qfrency, - formant_timbre, - proposal_pitch, - proposal_pitch_threshold, - embedders_mix, - embedders_mix_layers, - embedders_mix_ratio - ], - outputs=[ - cleaner, - autotune, - pitch, - clean_strength, - index_strength, - resample_sr, - filter_radius, - rms_mix_rate, - protect, - split_audio, - f0_autotune_strength, - formant_shifting, - formant_qfrency, - formant_timbre, - proposal_pitch, - proposal_pitch_threshold, - embedders_mix, - embedders_mix_layers, - embedders_mix_ratio - ] - ) - save_file_button.click( - fn=save_presets, - inputs=[ - name_to_save_file, - cleaner, - autotune, - pitch, - clean_strength, - index_strength, - resample_sr, - filter_radius, - rms_mix_rate, - protect, - split_audio, - f0_autotune_strength, - cleaner_checkbox, - autotune_checkbox, - pitch_checkbox, - index_strength_checkbox, - resample_sr_checkbox, - filter_radius_checkbox, - rms_mix_rate_checkbox, - protect_checkbox, - split_audio_checkbox, - formant_shifting_checkbox, - formant_shifting, - formant_qfrency, - formant_timbre, - proposal_pitch, - proposal_pitch_threshold, - embedders_mix, - embedders_mix_layers, - embedders_mix_ratio - ], - outputs=[ - presets_name - ] - ) - refresh_preset_button.click( - fn=change_preset_choices, - inputs=[], - outputs=[ - presets_name - ] - ) - with gr.Row(): - proposal_pitch.change( - fn=visible, - inputs=[ - proposal_pitch - ], - outputs=[ - proposal_pitch_threshold - ] - ) - upload_presets.upload( - fn=lambda presets_in: [ - shutil_move(preset.name, configs["presets_path"]) - for preset in presets_in - ][0], - inputs=[ - upload_presets - ], - outputs=[ - presets_name - ] - ) - translate_button.click( - fn=google_translate, - inputs=[ - prompt, - source_lang, - target_lang - ], - outputs=[ - prompt - ], - api_name="google_translate" - ) - with gr.Row(): - unlock_full_method.change( - fn=unlock_f0, - inputs=[ - unlock_full_method - ], - outputs=[ - f0_method - ] - ) - upload_f0_file.upload( - fn=lambda inp: shutil_move(inp.name, configs["f0_path"]), - inputs=[ - upload_f0_file - ], - outputs=[ - f0_file_dropdown - ] - ) - refresh_f0_file.click( - fn=change_f0_choices, - inputs=[], - outputs=[ - f0_file_dropdown - ] - ) - with gr.Row(): - embedder_mode.change( - fn=change_embedders_mode, - inputs=[ - embedder_mode - ], - outputs=[ - embedders - ] - ) - autotune.change( - fn=visible, - inputs=[ - autotune - ], - outputs=[ - f0_autotune_strength - ] - ) - model_pth.change( - fn=get_index, - inputs=[ - model_pth - ], - outputs=[ - model_index - ] - ) - with gr.Row(): - cleaner.change( - fn=visible, - inputs=[ - cleaner - ], - outputs=[ - clean_strength - ] - ) - f0_method.change( - fn=lambda method, hybrid: [ - visible(method == "hybrid"), - visible(method == "hybrid"), - hoplength_show(method, hybrid) - ], - inputs=[ - f0_method, - hybrid_f0method - ], - outputs=[ - hybrid_f0method, - alpha, - hop_length - ] - ) - hybrid_f0method.change( - fn=hoplength_show, - inputs=[ - f0_method, - hybrid_f0method - ], - outputs=[ - hop_length - ] - ) - with gr.Row(): - refresh.click( - fn=change_models_choices, - inputs=[], - outputs=[ - model_pth, - model_index - ] - ) - embedders.change( - fn=lambda embedders: visible(embedders == "custom"), - inputs=[ - embedders - ], - outputs=[ - custom_embedders - ] - ) - formant_shifting.change( - fn=lambda a: [ - visible(a) - for _ in range(2) - ], - inputs=[ - formant_shifting - ], - outputs=[ - formant_qfrency, - formant_timbre - ] - ) - with gr.Row(): - model_index.change( - fn=index_strength_show, - inputs=[ - model_index - ], - outputs=[ - index_strength, - nprobe - ] - ) - txt_input.upload( - fn=process_input, - inputs=[ - txt_input - ], - outputs=[ - prompt - ] - ) - use_txt.change( - fn=visible, - inputs=[ - use_txt - ], - outputs=[ - txt_input - ] - ) - use_txt.change( - fn=lambda: None, - inputs=[], - outputs=[ - txt_input - ] - ) - with gr.Row(): - google_tts_check_box.change( - fn=change_tts_voice_choices, - inputs=[ - google_tts_check_box - ], - outputs=[ - tts_voice - ] - ) - model_pth.change( - fn=get_speakers_id_and_architecture, - inputs=[ - model_pth - ], - outputs=[ - sids, - noise_scale - ] - ) - embedders_mix.change( - fn=visible, - inputs=[ - embedders_mix - ], - outputs=[ - embedders_mix_column - ] - ) - with gr.Row(): - tts_button.click( - fn=TTS, - inputs=[ - prompt, - tts_voice, - speed, - output_audio_tts, - tts_pitch, - google_tts_check_box, - txt_input - ], - outputs=[ - tts_voice_audio - ], - api_name="text-to-speech" - ) - convert_button.click( - fn=convert_tts, - inputs=[ - cleaner, - autotune, - pitch, - clean_strength, - model_pth, - model_index, - index_strength, - output_audio_tts, - output_audio, - export_format, - f0_method, - hybrid_f0method, - hop_length, - embedders, - custom_embedders, - resample_sr, - filter_radius, - rms_mix_rate, - protect, - split_audio, - f0_autotune_strength, - checkpointing, - predictor_onnx, - formant_shifting, - formant_qfrency, - formant_timbre, - f0_file_dropdown, - embedder_mode, - proposal_pitch, - proposal_pitch_threshold, - audio_processing, - alpha, - sids, - embedders_mix, - embedders_mix_layers, - embedders_mix_ratio, - noise_scale, - nprobe, - audio_upscaler - ], - outputs=[ - tts_voice_convert - ], - api_name="convert_tts" - ) \ No newline at end of file diff --git a/main/app/tabs/inference/child/convert_with_vad.py b/main/app/tabs/inference/child/convert_with_vad.py deleted file mode 100644 index 87c08450030215f28dffa9bf6116ed87ca992830..0000000000000000000000000000000000000000 --- a/main/app/tabs/inference/child/convert_with_vad.py +++ /dev/null @@ -1,810 +0,0 @@ -import os -import sys - -import gradio as gr - -sys.path.append(os.getcwd()) - -from main.app.core.inference import convert_with_vad - -from main.app.core.ui import ( - visible, - get_index, - unlock_f0, - shutil_move, - hoplength_show, - index_strength_show, - change_audios_choices, - change_models_choices, - change_embedders_mode, - get_speakers_id_and_architecture -) - -from main.app.variables import ( - configs, - method_f0, - model_name, - index_path, - file_types, - audio_params, - translations, - embedders_mode, - embedders_model, - paths_for_files, - hybrid_f0_method, - sample_rate_choice, - export_format_choices, -) - -def convert_with_vad_tab(): - with gr.Row(): - gr.Markdown(translations["convert_with_vad_info"]) - with gr.Row(): - with gr.Column(): - with gr.Group(): - with gr.Row(): - cleaner = gr.Checkbox( - label=translations["clear_audio"], - value=False, - interactive=True - ) - autotune = gr.Checkbox( - label=translations["autotune"], - value=False, - interactive=True - ) - checkpointing = gr.Checkbox( - label=translations["memory_efficient_training"], - value=False, - interactive=True - ) - formant_shifting = gr.Checkbox( - label=translations["formantshift"], - value=False, - interactive=True - ) - proposal_pitch = gr.Checkbox( - label=translations["proposal_pitch"], - value=False, - interactive=True - ) - audio_processing = gr.Checkbox( - label=translations["audio_processing"], - value=False, - interactive=True - ) - with gr.Row(): - num_spk = gr.Slider( - minimum=2, - maximum=8, - step=1, - info=translations["num_spk_info"], - label=translations["num_spk"], - value=2, - interactive=True - ) - with gr.Row(): - with gr.Column(): - convert_button = gr.Button( - translations["convert_audio"], - variant="primary" - ) - with gr.Row(): - with gr.Column(): - with gr.Accordion( - translations["model_accordion"] + " 1", - open=True - ): - with gr.Row(): - model_pth_1 = gr.Dropdown( - label=translations["model_name"], - choices=model_name, - value=model_name[0] if len(model_name) >= 1 else "", - interactive=True, - allow_custom_value=True - ) - model_index_1 = gr.Dropdown( - label=translations["index_path"], - choices=index_path, - value=index_path[0] if len(index_path) >= 1 else "", - interactive=True, - allow_custom_value=True - ) - with gr.Row(): - refresh_model_1 = gr.Button( - translations["refresh"] - ) - with gr.Row(): - pitch_1 = gr.Slider( - minimum=-24, - maximum=24, - step=1, - info=translations["pitch_info"], - label=translations["pitch"], - value=0, - interactive=True - ) - index_strength_1 = gr.Slider( - label=translations["index_strength"], - info=translations["index_strength_info"], - minimum=0, - maximum=1, - value=0.5, - step=0.01, - interactive=True, - visible=True - ) - with gr.Row(): - nprobe_1 = gr.Slider( - label=translations["nprobe"], - info=translations["nprobe_info"], - minimum=1, - maximum=64, - value=9, - step=1, - interactive=True, - visible=True - ) - with gr.Row(): - sid_dict_1, architecture_dict_1 = get_speakers_id_and_architecture(model_pth_1.value) - if sid_dict_1["visible"] == "hidden": sid_dict_1["visible"] = False - if architecture_dict_1["visible"] == "hidden": architecture_dict_1["visible"] = False - - sids_1 = gr.Dropdown( - label=translations["sids_label"], - info=translations["sids_info"], - choices=sid_dict_1["choices"], - value=sid_dict_1["value"], - visible=sid_dict_1["visible"], - interactive=True - ) - with gr.Row(): - noise_scale_1 = gr.Slider( - minimum=0.1, - maximum=1.0, - label=translations["noise_scale"], - info=translations["noise_scale_info"], - value=0.4, - step=0.01, - interactive=True, - visible=architecture_dict_1["visible"] - ) - with gr.Accordion( - translations["input_output"], - open=False - ): - with gr.Column(): - export_format = gr.Radio( - label=translations["export_format"], - info=translations["export_info"], - choices=export_format_choices, - value="wav", - interactive=True - ) - input_audio = gr.Dropdown( - label=translations["audio_path"], - value="", - choices=paths_for_files, - info=translations["provide_audio"], - allow_custom_value=True, - interactive=True - ) - output_audio = gr.Textbox( - label=translations["output_path"], - value="audios/output.wav", - placeholder="audios/output.wav", - info=translations["output_path_info"], - interactive=True - ) - with gr.Column(): - refresh_audio = gr.Button( - translations["refresh"] - ) - with gr.Row(): - drop_audio = gr.Files( - label=translations["drop_audio"], - file_types=file_types - ) - with gr.Column(): - with gr.Accordion( - translations["model_accordion"] + " 2", - open=True - ): - with gr.Row(): - model_pth_2 = gr.Dropdown( - label=translations["model_name"], - choices=model_name, - value=model_name[0] if len(model_name) >= 1 else "", - interactive=True, - allow_custom_value=True - ) - model_index_2 = gr.Dropdown( - label=translations["index_path"], - choices=index_path, - value=index_path[0] if len(index_path) >= 1 else "", - interactive=True, - allow_custom_value=True - ) - with gr.Row(): - refresh_model_2 = gr.Button( - translations["refresh"] - ) - with gr.Row(): - pitch_2 = gr.Slider( - minimum=-24, - maximum=24, - step=1, - info=translations["pitch_info"], - label=translations["pitch"], - value=0, - interactive=True - ) - index_strength_2 = gr.Slider( - label=translations["index_strength"], - info=translations["index_strength_info"], - minimum=0, - maximum=1, - value=0.5, - step=0.01, - interactive=True, - visible=True - ) - with gr.Row(): - nprobe_2 = gr.Slider( - label=translations["nprobe"], - info=translations["nprobe_info"], - minimum=1, - maximum=64, - value=9, - step=1, - interactive=True, - visible=True - ) - with gr.Row(): - sid_dict_2, architecture_dict_2 = get_speakers_id_and_architecture(model_pth_2.value) - if sid_dict_2["visible"] == "hidden": sid_dict_2["visible"] = False - if architecture_dict_2["visible"] == "hidden": architecture_dict_2["visible"] = False - - sids_2 = gr.Dropdown( - label=translations["sids_label"], - info=translations["sids_info"], - choices=sid_dict_2["choices"], - value=sid_dict_2["value"], - visible=sid_dict_2["visible"], - interactive=True - ) - with gr.Row(): - noise_scale_2 = gr.Slider( - minimum=0.1, - maximum=1.0, - label=translations["noise_scale"], - info=translations["noise_scale_info"], - value=0.4, - step=0.01, - interactive=True, - visible=architecture_dict_2["visible"] - ) - with gr.Accordion( - translations["setting"], - open=False - ): - with gr.Accordion( - translations["f0_method"], - open=False - ): - with gr.Group(): - with gr.Row(): - predictor_onnx = gr.Checkbox( - label=translations["predictor_onnx"], - info=translations["predictor_onnx_info"], - value=False, - interactive=True - ) - unlock_full_method = gr.Checkbox( - label=translations["f0_unlock"], - info=translations["f0_unlock_info"], - value=False, - interactive=True - ) - f0_method = gr.Radio( - label=translations["f0_method"], - info=translations["f0_method_info"], - choices=method_f0, - value="rmvpe", - interactive=True - ) - hybrid_f0method = gr.Dropdown( - label=translations["f0_method_hybrid"], - info=translations["f0_method_hybrid_info"], - choices=hybrid_f0_method, - value=hybrid_f0_method[0], - interactive=True, - allow_custom_value=True, - visible=False - ) - hop_length = gr.Slider( - label=translations['hop_length'], - info=translations["hop_length_info"], - minimum=64, - maximum=512, - value=160, - step=1, - interactive=True, - visible=False - ) - alpha = gr.Slider( - label=translations["alpha_label"], - info=translations["alpha_info"], - minimum=0.1, - maximum=1, - value=0.5, - step=0.1, - interactive=True, - visible=False - ) - with gr.Accordion( - translations["hubert_model"], - open=False - ): - embedders_mix = gr.Checkbox( - label=translations["embedders_mix"], - info=translations["embedders_mix_info"], - value=False, - interactive=True - ) - embedder_mode = gr.Radio( - label=translations["embed_mode"], - info=translations["embed_mode_info"], - value="fairseq", - choices=embedders_mode, - interactive=True, - visible=True - ) - embedders = gr.Radio( - label=translations["hubert_model"], - info=translations["hubert_info"], - choices=embedders_model, - value="hubert_base", - interactive=True - ) - custom_embedders = gr.Textbox( - label=translations["modelname"], - info=translations["modelname_info"], - value="", - placeholder="hubert_base", - interactive=True, - visible=False - ) - with gr.Column(visible=False) as embedders_mix_column: - embedders_mix_layers = gr.Slider( - label=translations["embedders_mix_layers"], - info=translations["embedders_mix_layers_info"], - minimum=1, - maximum=12, - value=9, - step=1, - interactive=True - ) - embedders_mix_ratio = gr.Slider( - label=translations["embedders_mix_ratio"], - info=translations["embedders_mix_ratio_info"], - minimum=0.1, - maximum=1, - value=0.5, - step=0.1, - interactive=True - ) - with gr.Column(): - resample_sr3 = gr.Radio( - choices=[0]+sample_rate_choice, - label=translations["resample"], - info=translations["resample_info"], - value=0, - interactive=True - ) - proposal_pitch_threshold = gr.Slider( - minimum=50.0, - maximum=1200.0, - label=translations["proposal_pitch_threshold"], - info=translations["proposal_pitch_threshold_info"], - value=255.0, - step=0.1, - interactive=True, - visible=False - ) - clean_strength = gr.Slider( - label=translations["clean_strength"], - info=translations["clean_strength_info"], - minimum=0, - maximum=1, - value=0.5, - step=0.1, - interactive=True, - visible=False - ) - f0_autotune_strength = gr.Slider( - minimum=0, - maximum=1, - label=translations["autotune_rate"], - info=translations["autotune_rate_info"], - value=1, - step=0.1, - interactive=True, - visible=False - ) - filter_radius = gr.Slider( - minimum=0, - maximum=7, - label=translations["filter_radius"], - info=translations["filter_radius_info"], - value=3, - step=1, - interactive=True - ) - rms_mix_rate = gr.Slider( - minimum=0, - maximum=1, - label=translations["rms_mix_rate"], - info=translations["rms_mix_rate_info"], - value=1, - step=0.1, - interactive=True - ) - protect = gr.Slider( - minimum=0, - maximum=1, - label=translations["protect"], - info=translations["protect_info"], - value=0.5, - step=0.01, - interactive=True - ) - with gr.Row(): - vad_sensitivity = gr.Slider( - minimum=0, - maximum=3, - label=translations["vad_sensitivity_label"], - info=translations["vad_sensitivity_info"], - value=3, - step=1, - interactive=True, - visible=False - ) - vad_frame_ms = gr.Slider( - minimum=10, - maximum=30, - label=translations["vad_frame_ms_label"], - info=translations["vad_frame_ms_info"], - value=30, - step=10, - interactive=True, - visible=False - ) - with gr.Row(): - formant_qfrency = gr.Slider( - value=1.0, - label=translations["formant_qfrency"] + " 1", - info=translations["formant_qfrency"], - minimum=0.0, - maximum=16.0, - step=0.1, - interactive=True, - visible=False - ) - formant_timbre = gr.Slider( - value=1.0, - label=translations["formant_timbre"] + " 1", - info=translations["formant_timbre"], - minimum=0.0, - maximum=16.0, - step=0.1, - interactive=True, - visible=False - ) - with gr.Row(): - formant_qfrency = gr.Slider( - value=1.0, - label=translations["formant_qfrency"] + " 2", - info=translations["formant_qfrency"], - minimum=0.0, - maximum=16.0, - step=0.1, - interactive=True, - visible=False - ) - formant_timbre = gr.Slider( - value=1.0, - label=translations["formant_timbre"] + " 2", - info=translations["formant_timbre"], - minimum=0.0, - maximum=16.0, - step=0.1, - interactive=True, - visible=False - ) - with gr.Row(): - gr.Markdown(translations["input_output"]) - with gr.Row(): - play_input_audio = gr.Audio( - interactive=False, - label=translations["input_audio"], - **audio_params - ) - play_output_audio = gr.Audio( - interactive=False, - label=translations["output_file_tts_convert"], - **audio_params - ) - with gr.Row(): - autotune.change( - fn=visible, - inputs=[ - autotune - ], - outputs=[ - f0_autotune_strength - ] - ) - cleaner.change( - fn=visible, - inputs=[ - cleaner - ], - outputs=[ - clean_strength - ] - ) - f0_method.change( - fn=lambda method, hybrid: [ - visible(method == "hybrid"), - visible(method == "hybrid"), - hoplength_show(method, hybrid) - ], - inputs=[ - f0_method, - hybrid_f0method - ], - outputs=[ - hybrid_f0method, - alpha, - hop_length - ] - ) - with gr.Row(): - hybrid_f0method.change( - fn=hoplength_show, - inputs=[ - f0_method, - hybrid_f0method - ], - outputs=[ - hop_length - ] - ) - refresh_model_1.click( - fn=change_models_choices, - inputs=[], - outputs=[ - model_pth_1, - model_index_1 - ] - ) - model_pth_1.change( - fn=get_index, - inputs=[ - model_pth_1 - ], - outputs=[ - model_index_1 - ] - ) - with gr.Row(): - refresh_model_2.click( - fn=change_models_choices, - inputs=[], - outputs=[ - model_pth_2, - model_index_2 - ] - ) - model_pth_2.change( - fn=get_index, - inputs=[ - model_pth_2 - ], - outputs=[ - model_index_2 - ] - ) - drop_audio.upload( - fn=lambda audio_in: [ - shutil_move(audio.name, configs["audios_path"]) - for audio in audio_in - ][0], - inputs=[ - drop_audio - ], - outputs=[ - input_audio - ] - ) - with gr.Row(): - input_audio.change( - fn=lambda audio: audio if os.path.isfile(audio) else None, - inputs=[ - input_audio - ], - outputs=[ - play_input_audio - ] - ) - formant_shifting.change( - fn=lambda a: [ - visible(a) - for _ in range(4) - ], - inputs=[ - formant_shifting - ], - outputs=[ - formant_qfrency, - formant_timbre, - formant_qfrency, - formant_timbre - ] - ) - embedders.change( - fn=lambda embedders: visible(embedders == "custom"), - inputs=[ - embedders - ], - outputs=[ - custom_embedders - ] - ) - with gr.Row(): - refresh_audio.click( - fn=change_audios_choices, - inputs=[ - input_audio - ], - outputs=[ - input_audio - ] - ) - model_index_1.change( - fn=index_strength_show, - inputs=[ - model_index_1 - ], - outputs=[ - index_strength_1, - nprobe_1 - ] - ) - model_index_2.change( - fn=index_strength_show, - inputs=[ - model_index_2 - ], - outputs=[ - index_strength_2, - nprobe_2 - ] - ) - with gr.Row(): - unlock_full_method.change( - fn=unlock_f0, - inputs=[ - unlock_full_method - ], - outputs=[ - f0_method - ] - ) - embedder_mode.change( - fn=change_embedders_mode, - inputs=[ - embedder_mode - ], - outputs=[ - embedders - ] - ) - proposal_pitch.change( - fn=visible, - inputs=[ - proposal_pitch - ], - outputs=[ - proposal_pitch_threshold - ] - ) - with gr.Row(): - model_pth_1.change( - fn=get_speakers_id_and_architecture, - inputs=[ - model_pth_1 - ], - outputs=[ - sids_1, - noise_scale_1 - ] - ) - model_pth_2.change( - fn=get_speakers_id_and_architecture, - inputs=[ - model_pth_2 - ], - outputs=[ - sids_2, - noise_scale_2 - ] - ) - embedders_mix.change( - fn=visible, - inputs=[ - embedders_mix - ], - outputs=[ - embedders_mix_column - ] - ) - with gr.Row(): - convert_button.click( - fn=convert_with_vad, - inputs=[ - num_spk, - cleaner, - clean_strength, - autotune, - f0_autotune_strength, - checkpointing, - model_pth_1, - model_pth_2, - model_index_1, - model_index_2, - pitch_1, - pitch_2, - index_strength_1, - index_strength_2, - export_format, - input_audio, - output_audio, - predictor_onnx, - f0_method, - hybrid_f0method, - hop_length, - embedder_mode, - embedders, - custom_embedders, - resample_sr3, - filter_radius, - rms_mix_rate, - protect, - formant_shifting, - formant_qfrency, - formant_timbre, - formant_qfrency, - formant_timbre, - proposal_pitch, - proposal_pitch_threshold, - audio_processing, - alpha, - sids_1, - sids_2, - embedders_mix, - embedders_mix_layers, - embedders_mix_ratio, - noise_scale_1, - noise_scale_2, - vad_sensitivity, - vad_frame_ms, - nprobe_1, - nprobe_2 - ], - outputs=[ - play_output_audio - ], - api_name="convert_with_vad" - ) \ No newline at end of file diff --git a/main/app/tabs/inference/child/separate.py b/main/app/tabs/inference/child/separate.py deleted file mode 100644 index 9e02642239424f1433ef19e28cca727db48a8d65..0000000000000000000000000000000000000000 --- a/main/app/tabs/inference/child/separate.py +++ /dev/null @@ -1,600 +0,0 @@ -import os -import sys - -import gradio as gr - -sys.path.append(os.getcwd()) - -from main.app.core.downloads import download_url -from main.app.core.separate import separate_music - -from main.app.core.ui import ( - visible, - shutil_move, - separate_change, - change_audios_choices, - valueFalse_interactive -) - -from main.app.variables import ( - configs, - uvr_model, - vr_models, - mdx_models, - file_types, - audio_params, - translations, - reverb_models, - karaoke_models, - denoise_models, - paths_for_files, - sample_rate_choice, - export_format_choices -) - -def separate_tab(): - with gr.Row(): - gr.Markdown(translations["4_part"]) - with gr.Row(): - with gr.Column(): - with gr.Group(): - with gr.Row(): - enable_denoise = gr.Checkbox( - label=translations["denoise_mdx"], - value=False, - interactive=False - ) - separate_backing = gr.Checkbox( - label=translations["separator_backing"], - value=False, - interactive=True - ) - separate_reverb = gr.Checkbox( - label=translations["dereveb_audio"], - value=False, - interactive=True - ) - enable_tta = gr.Checkbox( - label=translations["enable_tta"], - value=False, - interactive=False - ) - high_end_process = gr.Checkbox( - label=translations["high_end_process"], - value=False, - interactive=False - ) - enable_post_process = gr.Checkbox( - label=translations["enable_post_process"], - value=False, - interactive=False - ) - with gr.Row(): - model_name = gr.Dropdown( - label=translations["separator_model"], - value=uvr_model[0], - choices=uvr_model, - interactive=True - ) - karaoke_model = gr.Dropdown( - label=translations["separator_backing_model"], - value=list(karaoke_models.keys())[0], - choices=list(karaoke_models.keys()), - interactive=True, - visible=False - ) - reverb_model = gr.Dropdown( - label=translations["dereveb_model"], - value=list(reverb_models.keys())[0], - choices=list(reverb_models.keys()), - interactive=True, - visible=False - ) - denoise_model = gr.Dropdown( - label=translations["denoise_model"], - value=list(denoise_models.keys())[0], - choices=list(denoise_models.keys()), - interactive=True, - visible=False - ) - with gr.Row(): - with gr.Column(): - separate_button = gr.Button( - translations["separator_tab"], - variant="primary" - ) - with gr.Row(): - with gr.Column(): - with gr.Group(): - with gr.Row(): - shifts = gr.Slider( - label=translations["shift"], - info=translations["shift_info"], - minimum=1, - maximum=20, - value=2, - step=1, - interactive=True - ) - batch_size = gr.Slider( - label=translations["batch_size"], - info=translations["mdx_batch_size_info"], - minimum=1, - maximum=64, - value=1, - step=1, - interactive=True, - visible=False - ) - with gr.Row(): - segments_size = gr.Slider( - label=translations["segments_size"], - info=translations["segments_size_info"], - minimum=32, - maximum=3072, - value=256, - step=32, - interactive=True - ) - aggression = gr.Slider( - label=translations['aggression'], - info=translations["aggression_info"], - minimum=1, - maximum=50, - value=5, - step=1, - interactive=True, - visible=False - ) - drop_audio = gr.Files( - label=translations["drop_audio"], - file_types=file_types - ) - with gr.Accordion( - translations["use_url"], - open=False - ): - url = gr.Textbox( - label=translations["url_audio"], - value="", - placeholder="https://www.youtube.com/...", - scale=6 - ) - download_button = gr.Button( - translations["downloads"] - ) - with gr.Column(): - with gr.Group(): - with gr.Row(): - overlap = gr.Radio( - label=translations["overlap"], - info=translations["overlap_info"], - choices=[ - "0.25", - "0.5", - "0.75", - "0.99" - ], - value="0.25", - interactive=True - ) - with gr.Row(): - window_size = gr.Slider( - label=translations["window_size"], - info=translations["window_size_info"], - minimum=320, - maximum=1024, - value=512, - step=32, - interactive=True, - visible=False - ) - hop_length = gr.Slider( - label=translations['hop_length'], - info=translations["hop_length_info"], - minimum=64, - maximum=8192, - value=1024, - step=1, - interactive=True, - visible=False - ) - post_process_threshold = gr.Slider( - label=translations['post_process_threshold'], - info=translations["post_process_threshold_info"], - minimum=0.1, - maximum=0.3, - value=0.2, - step=0.1, - interactive=True, - visible=False - ) - sample_rate = gr.Radio( - choices=sample_rate_choice, - value=44100, - label=translations["sr"], - info=translations["sr_info"], - interactive=True - ) - with gr.Accordion( - translations["input_output"], - open=False - ): - export_format = gr.Radio( - label=translations["export_format"], - info=translations["export_info"], - choices=export_format_choices, - value="wav", - interactive=True - ) - input_audio = gr.Dropdown( - label=translations["audio_path"], - value="", - choices=paths_for_files, - allow_custom_value=True, - interactive=True - ) - refresh_audio = gr.Button( - translations["refresh"] - ) - output_dirs = gr.Textbox( - label=translations["output_folder"], - value="audios", - placeholder="audios", - info=translations["output_folder_info"], - interactive=True - ) - audio_input = gr.Audio( - interactive=False, - label=translations["input_audio"], - **audio_params - ) - with gr.Row(): - gr.Markdown(translations["output_separator"]) - with gr.Row(): - instruments_audio = gr.Audio( - interactive=False, - label=translations["instruments"], - visible=True, - **audio_params - ) - original_vocals = gr.Audio( - interactive=False, - label=translations["original_vocal"], - visible=True, - **audio_params - ) - main_vocals = gr.Audio( - interactive=False, - label=translations["main_vocal"], - visible=False, - **audio_params - ) - backing_vocals = gr.Audio( - interactive=False, - label=translations["backing_vocal"], - visible=False, - **audio_params - ) - with gr.Row(): - model_name.change( - fn=lambda a: valueFalse_interactive(a in list(mdx_models.keys()) + list(vr_models.keys())), - inputs=[ - model_name - ], - outputs=[ - enable_denoise - ] - ) - separate_backing.change( - fn=lambda a, b: valueFalse_interactive(a or b), - inputs=[ - separate_backing, - separate_reverb - ], - outputs=[ - enable_denoise - ] - ) - separate_reverb.change( - fn=lambda a, b: valueFalse_interactive(a or b), - inputs=[ - separate_backing, - separate_reverb - ], - outputs=[ - enable_denoise - ] - ) - with gr.Row(): - input_audio.change( - fn=lambda audio: audio if os.path.isfile(audio) else None, - inputs=[ - input_audio - ], - outputs=[ - audio_input - ] - ) - drop_audio.upload( - fn=lambda audio_in: [ - shutil_move(audio.name, configs["audios_path"]) - for audio in audio_in - ][0], - inputs=[ - drop_audio - ], - outputs=[ - input_audio - ] - ) - refresh_audio.click( - fn=change_audios_choices, - inputs=[ - input_audio - ], - outputs=[ - input_audio - ] - ) - with gr.Row(): - separate_backing.change( - fn=lambda a: [ - visible(a) - for _ in range(2) - ], - inputs=[ - separate_backing - ], - outputs=[ - main_vocals, - backing_vocals - ] - ) - download_button.click( - fn=download_url, - inputs=[ - url - ], - outputs=[ - input_audio, - audio_input, - url - ], - api_name='download_url' - ) - with gr.Row(): - model_name.change( - fn=separate_change, - inputs=[ - model_name, - karaoke_model, - reverb_model, - enable_post_process, - separate_backing, - separate_reverb, - enable_denoise - ], - outputs=[ - karaoke_model, - reverb_model, - overlap, - segments_size, - hop_length, - batch_size, - shifts, - window_size, - aggression, - post_process_threshold, - denoise_model, - enable_tta, - high_end_process, - enable_post_process, - ] - ) - with gr.Row(): - karaoke_model.change( - fn=separate_change, - inputs=[ - model_name, - karaoke_model, - reverb_model, - enable_post_process, - separate_backing, - separate_reverb, - enable_denoise - ], - outputs=[ - karaoke_model, - reverb_model, - overlap, - segments_size, - hop_length, - batch_size, - shifts, - window_size, - aggression, - post_process_threshold, - denoise_model, - enable_tta, - high_end_process, - enable_post_process, - ] - ) - separate_backing.change( - fn=separate_change, - inputs=[ - model_name, - karaoke_model, - reverb_model, - enable_post_process, - separate_backing, - separate_reverb, - enable_denoise - ], - outputs=[ - karaoke_model, - reverb_model, - overlap, - segments_size, - hop_length, - batch_size, - shifts, - window_size, - aggression, - post_process_threshold, - denoise_model, - enable_tta, - high_end_process, - enable_post_process, - ] - ) - with gr.Row(): - reverb_model.change( - fn=separate_change, - inputs=[ - model_name, - karaoke_model, - reverb_model, - enable_post_process, - separate_backing, - separate_reverb, - enable_denoise - ], - outputs=[ - karaoke_model, - reverb_model, - overlap, - segments_size, - hop_length, - batch_size, - shifts, - window_size, - aggression, - post_process_threshold, - denoise_model, - enable_tta, - high_end_process, - enable_post_process, - ] - ) - separate_reverb.change( - fn=separate_change, - inputs=[ - model_name, - karaoke_model, - reverb_model, - enable_post_process, - separate_backing, - separate_reverb, - enable_denoise - ], - outputs=[ - karaoke_model, - reverb_model, - overlap, - segments_size, - hop_length, - batch_size, - shifts, - window_size, - aggression, - post_process_threshold, - denoise_model, - enable_tta, - high_end_process, - enable_post_process, - ] - ) - with gr.Row(): - enable_denoise.change( - fn=separate_change, - inputs=[ - model_name, - karaoke_model, - reverb_model, - enable_post_process, - separate_backing, - separate_reverb, - enable_denoise - ], - outputs=[ - karaoke_model, - reverb_model, - overlap, - segments_size, - hop_length, - batch_size, - shifts, - window_size, - aggression, - post_process_threshold, - denoise_model, - enable_tta, - high_end_process, - enable_post_process, - ] - ) - enable_post_process.change( - fn=separate_change, - inputs=[ - model_name, - karaoke_model, - reverb_model, - enable_post_process, - separate_backing, - separate_reverb, - enable_denoise - ], - outputs=[ - karaoke_model, - reverb_model, - overlap, - segments_size, - hop_length, - batch_size, - shifts, - window_size, - aggression, - post_process_threshold, - denoise_model, - enable_tta, - high_end_process, - enable_post_process, - ] - ) - with gr.Row(): - separate_button.click( - fn=separate_music, - inputs=[ - input_audio, - output_dirs, - export_format, - model_name, - karaoke_model, - reverb_model, - denoise_model, - sample_rate, - shifts, - batch_size, - overlap, - aggression, - hop_length, - window_size, - segments_size, - post_process_threshold, - enable_tta, - enable_denoise, - high_end_process, - enable_post_process, - separate_backing, - separate_reverb - ], - outputs=[ - original_vocals, - instruments_audio, - main_vocals, - backing_vocals - ], - api_name="separate_music" - ) \ No newline at end of file diff --git a/main/app/tabs/inference/inference.py b/main/app/tabs/inference/inference.py deleted file mode 100644 index c477e6d777256245b87c328f6bf0cee3263578f0..0000000000000000000000000000000000000000 --- a/main/app/tabs/inference/inference.py +++ /dev/null @@ -1,41 +0,0 @@ -import os -import sys - -import gradio as gr - -sys.path.append(os.getcwd()) - -from main.app.variables import ( - configs, - translations -) - -def inference_tab(): - with gr.TabItem(translations["inference"]): - if configs.get("separator_tab", True): - from main.app.tabs.inference.child.separate import separate_tab - - with gr.TabItem(translations["separator_tab"]): - gr.Markdown(f"## {translations['separator_tab']}") - separate_tab() - - if configs.get("convert_tab", True): - from main.app.tabs.inference.child.convert import convert_tab - - with gr.TabItem(translations["convert_audio"]): - gr.Markdown(f"## {translations['convert_audio']}") - convert_tab() - - if configs.get("convert_with_vad_tab", True): - from main.app.tabs.inference.child.convert_with_vad import convert_with_vad_tab - - with gr.TabItem(translations["convert_with_vad"]): - gr.Markdown(f"## {translations['convert_with_vad']}") - convert_with_vad_tab() - - if configs.get("tts_tab", True): - from main.app.tabs.inference.child.convert_tts import convert_tts_tab - - with gr.TabItem(translations["convert_text"]): - gr.Markdown(translations["convert_text_markdown"]) - convert_tts_tab() diff --git a/main/app/tabs/realtime/realtime.py b/main/app/tabs/realtime/realtime.py deleted file mode 100644 index 70492eb7753f37c4e023879c4f14bcf147a17b86..0000000000000000000000000000000000000000 --- a/main/app/tabs/realtime/realtime.py +++ /dev/null @@ -1,2457 +0,0 @@ -import os -import sys - -import gradio as gr - -sys.path.append(os.getcwd()) - -from main.app.core.realtime import ( - realtime_stop, - change_config, - realtime_start, - soundfile_record_audio, -) - -from main.app.variables import ( - configs, - method_f0, - model_name, - index_path, - audio_params, - translations, - embedders_mode, - embedders_model, - sample_rate_choice, - export_format_choices, - realtime_presets_file -) - -from main.app.core.ui import ( - visible, - get_index, - unlock_f0, - shutil_move, - visibleFalse, - hoplength_show, - input_channels_map, - output_channels_map, - update_audio_device, - index_strength_show, - change_models_choices, - change_embedders_mode, - update_value_from_json, - update_button_from_json, - update_dropdowns_from_json, - change_audio_device_choices, - change_realtime_preset_choices, - get_speakers_id_and_architecture -) - -from main.app.core.presets import ( - save_realtime_config, - load_realtime_config -) - -client_mode = True # "--client" in sys.argv - -def realtime_tab(): - with gr.TabItem(translations["realtime"]): - gr.Markdown(translations["realtime_markdown"]) - with gr.Row(): - gr.Markdown(translations["realtime_markdown_2"]) - with gr.Row(): - status = gr.Label( - label=translations["realtime_latency"], - value=translations["realtime_not_startup"], - elem_id="realtime-status-info" - ) - with gr.Row(): - monitor = gr.Checkbox( - label=translations["monitor"], - value=False, - interactive=True - ) - exclusive_mode = gr.Checkbox( - label=translations["exclusive_mode"], - value=False, - interactive=True - ) - vad_enabled = gr.Checkbox( - label=translations["vad_enabled"], - value=False, - interactive=True - ) - clean_audio = gr.Checkbox( - label=translations["clear_audio"], - value=False, - interactive=True - ) - with gr.Row(): - with gr.Accordion( - translations["audio_device"], - open=True - ): - with gr.Row(): - input_audio_device = gr.Dropdown( - label=translations["input_audio_device_label"], - info=translations["input_audio_device_info"], - choices=list(input_channels_map.keys()) if not client_mode else [], - value=(list(input_channels_map.keys())[0] if len(list(input_channels_map.keys())) >= 1 else "") if not client_mode else None, - interactive=True - ) - output_audio_device = gr.Dropdown( - label=translations["output_audio_device_label"], - info=translations["output_audio_device_info"], - choices=list(output_channels_map.keys()) if not client_mode else [], - value=(list(output_channels_map.keys())[0] if len(list(output_channels_map.keys())) >= 1 else "") if not client_mode else None, - interactive=True - ) - monitor_output_device = gr.Dropdown( - label=translations["monitor_output_device_label"], - info=translations["monitor_output_device_info"], - choices=list(output_channels_map.keys()) if not client_mode else [], - value=(list(output_channels_map.keys())[0] if len(list(output_channels_map.keys())) >= 1 else "") if not client_mode else None, - interactive=True, - visible=False - ) - with gr.Row(): - input_audio_gain = gr.Slider( - minimum=0, - maximum=2500, - label=translations["input_audio_gain_label"], - info=translations["input_audio_gain_info"], - value=100, - step=1, - interactive=True - ) - output_audio_gain = gr.Slider( - minimum=0, - maximum=4000, - label=translations["output_audio_gain_label"], - info=translations["output_audio_gain_info"], - value=100, - step=1, - interactive=True - ) - monitor_audio_gain = gr.Slider( - minimum=0, - maximum=4000, - label=translations["monitor_audio_gain_label"], - info=translations["monitor_audio_gain_info"], - value=100, - step=1, - interactive=True, - visible=False - ) - with gr.Row(visible=not client_mode): - input_audio_sample_rate = gr.Radio( - choices=sample_rate_choice, - value=48000, - label=translations["samplerate"].split("|")[0], - info=translations["samplerate_info"], - interactive=True - ) - output_audio_sample_rate = gr.Radio( - choices=sample_rate_choice, - value=48000, - label=translations["samplerate"].split("|")[1], - info=translations["samplerate_info"], - interactive=True - ) - monitor_audio_sample_rate = gr.Radio( - choices=sample_rate_choice, - value=48000, - label=translations["samplerate"].split("|")[2], - info=translations["samplerate_info"], - interactive=True, - visible=False - ) - with gr.Row(): - asio_output_stereo = gr.Checkbox( - label=translations["stereo_device"].split("|")[0], - value=True, - interactive=True, - visible=False - ) - asio_monitor_stereo = gr.Checkbox( - label=translations["stereo_device"].split("|")[1], - value=True, - interactive=True, - visible=False - ) - with gr.Row(visible=False) as asio_row: - input_asio_channels = gr.Slider( - minimum=-1, - maximum=128, - label=translations["input_asio_channels_label"], - info=translations["input_asio_channels_info"], - value=0, - step=1, - interactive=True, - visible=False - ) - output_asio_channels = gr.Slider( - minimum=-1, - maximum=128, - label=translations["output_asio_channels_label"], - info=translations["output_asio_channels_info"], - value=0, - step=1, - interactive=True, - visible=False - ) - monitor_asio_channels = gr.Slider( - minimum=-1, - maximum=128, - label=translations["monitor_asio_channels_label"], - info=translations["monitor_asio_channels_info"], - value=0, - step=1, - interactive=True, - visible=False - ) - with gr.Row(): - refresh_audio_device = gr.Button( - value=translations["refresh_audio_device"], - variant="secondary" - ) - with gr.Row(): - start_realtime = gr.Button( - value=translations["start_realtime_button"], - variant="primary", - interactive=True - ) - stop_realtime = gr.Button( - value=translations["stop_realtime_button"], - variant="stop", - interactive=False - ) - with gr.Row(): - with gr.Accordion(translations["record"], open=False): - with gr.Column(): - record_audio_path = gr.Textbox( - label=translations["output_path"], - placeholder="audios/record_audio.wav", - info=translations["output_path_info"], - value="audios/record_audio.wav", - interactive=True, - ) - export_format = gr.Radio( - label=translations["export_format"], - info=translations["export_info"], - choices=export_format_choices, - value="wav", - interactive=True, - ) - record_audio = gr.Button( - translations["start_record"] - ) - record_output = gr.Audio( - label=translations["output_audio"], - **audio_params - ) - with gr.Row(): - chunk_size = gr.Slider( - minimum=2.7, - maximum=2730.7, - step=0.1, - label=translations["chunk_size"], - info=translations["chunk_size_info"], - value=1024, - interactive=True - ) - pitch = gr.Slider( - minimum=-24, - maximum=24, - step=1, - info=translations["pitch_info"], - label=translations["pitch"], - value=0, - interactive=True - ) - with gr.Row(): - with gr.Column(): - with gr.Accordion( - translations["model_accordion"], - open=True - ): - with gr.Row(): - model_pth = gr.Dropdown( - label=translations["model_name"], - choices=model_name, - value=model_name[0] if len(model_name) >= 1 else "", - interactive=True, - allow_custom_value=True - ) - model_index = gr.Dropdown( - label=translations["index_path"], - choices=index_path, - value=index_path[0] if len(index_path) >= 1 else "", - interactive=True, - allow_custom_value=True - ) - with gr.Row(): - model_refresh = gr.Button( - translations["refresh"] - ) - with gr.Row(): - index_strength = gr.Slider( - label=translations["index_strength"], - info=translations["index_strength_info"], - minimum=0, - maximum=1, - value=0.5, - step=0.01, - interactive=True, - visible=True - ) - nprobe = gr.Slider( - label=translations["nprobe"], - info=translations["nprobe_info"], - minimum=1, - maximum=64, - value=9, - step=1, - interactive=True, - visible=True - ) - with gr.Row(): - sid_dict, architecture_dict = get_speakers_id_and_architecture(model_pth.value) - if sid_dict["visible"] == "hidden": sid_dict["visible"] = False - if architecture_dict["visible"] == "hidden": architecture_dict["visible"] = False - - sids = gr.Dropdown( - label=translations["sids_label"], - info=translations["sids_info"], - choices=sid_dict["choices"], - value=sid_dict["value"], - visible=sid_dict["visible"], - interactive=True - ) - with gr.Column(): - with gr.Accordion( - translations["f0_method"], - open=True - ): - with gr.Group(): - with gr.Row(): - predictor_onnx = gr.Checkbox( - label=translations["predictor_onnx"], - info=translations["predictor_onnx_info"], - value=False, - interactive=True - ) - unlock_full_method = gr.Checkbox( - label=translations["f0_unlock"], - info=translations["f0_unlock_info"], - value=False, - interactive=True - ) - f0_method = gr.Radio( - label=translations["f0_method"], - info=translations["f0_method_info"], - choices=[m for m in method_f0 if m != "hybrid"], - value="rmvpe", - interactive=True - ) - hop_length = gr.Slider( - label=translations['hop_length'], - info=translations["hop_length_info"], - minimum=64, - maximum=512, - value=160, - step=1, - interactive=True, - visible=False - ) - with gr.Column(): - with gr.Accordion( - translations["hubert_model"], - open=True - ): - embedders_mix = gr.Checkbox( - label=translations["embedders_mix"], - info=translations["embedders_mix_info"], - value=False, - interactive=True - ) - embedder_mode = gr.Radio( - label=translations["embed_mode"], - info=translations["embed_mode_info"], - value="fairseq", - choices=embedders_mode, - interactive=True, - visible=True) - embedders = gr.Radio( - label=translations["hubert_model"], - info=translations["hubert_info"], - choices=embedders_model, - value="hubert_base", - interactive=True - ) - custom_embedders = gr.Textbox( - label=translations["modelname"], - info=translations["modelname_info"], - value="", - placeholder="hubert_base", - interactive=True, - visible=False - ) - with gr.Column(visible=False) as embedders_mix_column: - embedders_mix_layers = gr.Slider( - label=translations["embedders_mix_layers"], - info=translations["embedders_mix_layers_info"], - minimum=1, - maximum=12, - value=9, - step=1, - interactive=True - ) - embedders_mix_ratio = gr.Slider( - label=translations["embedders_mix_ratio"], - info=translations["embedders_mix_ratio_info"], - minimum=0.1, - maximum=1, - value=0.5, - step=0.1, - interactive=True - ) - with gr.Row(): - with gr.Accordion( - translations["setting"], - open=True - ): - with gr.Row(): - f0_autotune = gr.Checkbox( - label=translations["autotune"], - value=False, - interactive=True - ) - proposal_pitch = gr.Checkbox( - label=translations["proposal_pitch"], - value=False, - interactive=True - ) - with gr.Group(): - with gr.Row(): - f0_autotune_strength = gr.Slider( - minimum=0, - maximum=1, - label=translations["autotune_rate"], - info=translations["autotune_rate_info"], - value=1, - step=0.1, - interactive=True, - visible=False - ) - proposal_pitch_threshold = gr.Slider( - minimum=50.0, - maximum=1200.0, - label=translations["proposal_pitch_threshold"], - info=translations["proposal_pitch_threshold_info"], - value=255.0, - step=0.1, - interactive=True, - visible=False - ) - with gr.Row(): - rms_mix_rate = gr.Slider( - minimum=0, - maximum=1, - label=translations["rms_mix_rate"], - info=translations["rms_mix_rate_info"], - value=1, - step=0.1, - interactive=True - ) - protect = gr.Slider( - minimum=0, - maximum=1, - label=translations["protect"], - info=translations["protect_info"], - value=0.5, - step=0.01, - interactive=True - ) - with gr.Row(): - clean_strength = gr.Slider( - label=translations["clean_strength"], - info=translations["clean_strength_info"], - minimum=0, - maximum=1, - value=0.5, - step=0.1, - interactive=True, - visible=False - ) - filter_radius = gr.Slider( - minimum=0, - maximum=7, - label=translations["filter_radius"], - info=translations["filter_radius_info"], - value=3, - step=1, - interactive=True - ) - with gr.Column(): - silent_threshold = gr.Slider( - minimum=-90, - maximum=-60, - label=translations["silent_threshold_label"], - info=translations["silent_threshold_info"], - value=-90, - step=1, - interactive=True - ) - extra_convert_size = gr.Slider( - minimum=0.1, - maximum=5, - label=translations["extra_convert_size_label"], - info=translations["extra_convert_size_info"], - value=0.5, - step=0.1, - interactive=True - ) - cross_fade_overlap_size = gr.Slider( - minimum=0.05, - maximum=0.2, - label=translations["cross_fade_overlap_size_label"], - info=translations["cross_fade_overlap_size_info"], - value=0.1, - step=0.01, - interactive=True - ) - noise_scale = gr.Slider( - minimum=0.1, - maximum=1.0, - label=translations["noise_scale"], - info=translations["noise_scale_info"], - value=0.4, - step=0.01, - interactive=True, - visible=architecture_dict["visible"] - ) - with gr.Row(): - vad_sensitivity = gr.Slider( - minimum=0, - maximum=3, - label=translations["vad_sensitivity_label"], - info=translations["vad_sensitivity_info"], - value=3, - step=1, - interactive=True, - visible=False - ) - vad_frame_ms = gr.Slider( - minimum=10, - maximum=30, - label=translations["vad_frame_ms_label"], - info=translations["vad_frame_ms_info"], - value=30, - step=10, - interactive=True, - visible=False - ) - with gr.Row(): - post_process = gr.Checkbox( - label=translations["audio_effects"], - value=False, - interactive=True - ) - reverb = gr.Checkbox( - label=translations["reverb"], - value=False, - interactive=True, - visible=False - ) - chorus = gr.Checkbox( - label=translations["chorus"], - value=False, - interactive=True, - visible=False - ) - delay = gr.Checkbox( - label=translations["delay"], - value=False, - interactive=True, - visible=False - ) - phaser = gr.Checkbox( - label=translations["phaser"], - value=False, - interactive=True, - visible=False - ) - compressor = gr.Checkbox( - label=translations["compressor"], - value=False, - interactive=True, - visible=False - ) - with gr.Row(): - limiter = gr.Checkbox( - label=translations["limiter"], - value=False, - interactive=True, - visible=False - ) - distortion = gr.Checkbox( - label=translations["distortion"], - value=False, - interactive=True, - visible=False - ) - pitch_shift = gr.Checkbox( - label=translations["pitch"], - value=False, - interactive=True, - visible=False - ) - gain = gr.Checkbox( - label=translations["gain"], - value=False, - interactive=True, - visible=False - ) - bitcrush = gr.Checkbox( - label=translations["bitcrush"], - value=False, - interactive=True, - visible=False - ) - clipping = gr.Checkbox( - label=translations["clipping"], - value=False, - interactive=True, - visible=False - ) - with gr.Accordion( - translations["reverb"], - open=True, - visible=False - ) as reverb_accordion: - reverb_freeze_mode = gr.Checkbox( - label=translations["reverb_freeze"], - info=translations["reverb_freeze_info"], - value=False, - interactive=True - ) - reverb_room_size = gr.Slider( - minimum=0, - maximum=1, - step=0.01, - value=0.15, - label=translations["room_size"], - info=translations["room_size_info"], - interactive=True - ) - reverb_damping = gr.Slider( - minimum=0, - maximum=1, - step=0.01, - value=0.7, - label=translations["damping"], - info=translations["damping_info"], - interactive=True - ) - reverb_wet_gain = gr.Slider( - minimum=0, - maximum=1, - step=0.01, - value=0.2, - label=translations["wet_level"], - info=translations["wet_level_info"], - interactive=True - ) - reverb_dry_gain = gr.Slider( - minimum=0, - maximum=1, - step=0.01, - value=0.8, - label=translations["dry_level"], - info=translations["dry_level_info"], - interactive=True - ) - reverb_width = gr.Slider( - minimum=0, - maximum=1, - step=0.01, - value=1, - label=translations["width"], - info=translations["width_info"], - interactive=True - ) - with gr.Accordion( - translations["chorus"], - open=True, - visible=False - ) as chorus_accordion: - chorus_depth = gr.Slider( - minimum=0, - maximum=1, - step=0.01, - value=0.5, - label=translations["chorus_depth"], - info=translations["chorus_depth_info"], - interactive=True - ) - chorus_rate = gr.Slider( - minimum=0.1, - maximum=10, - step=0.1, - value=1.5, - label=translations["chorus_rate_hz"], - info=translations["chorus_rate_hz_info"], - interactive=True - ) - chorus_mix = gr.Slider( - minimum=0, - maximum=1, - step=0.01, - value=0.5, - label=translations["chorus_mix"], - info=translations["chorus_mix_info"], - interactive=True - ) - chorus_center_delay = gr.Slider( - minimum=0, - maximum=50, - step=1, - value=10, - label=translations["chorus_center_delay_ms"], - info=translations["chorus_center_delay_ms_info"], - interactive=True - ) - chorus_feedback = gr.Slider( - minimum=-1, - maximum=1, - step=0.01, - value=0, - label=translations["chorus_feedback"], - info=translations["chorus_feedback_info"], - interactive=True - ) - with gr.Accordion( - translations["phaser"], - open=True, - visible=False - ) as phaser_accordion: - phaser_depth = gr.Slider( - minimum=0, - maximum=1, - step=0.01, - value=0.5, - label=translations["phaser_depth"], - info=translations["phaser_depth_info"], - interactive=True - ) - phaser_rate_hz = gr.Slider( - minimum=0.1, - maximum=10, - step=0.1, - value=1, - label=translations["phaser_rate_hz"], - info=translations["phaser_rate_hz_info"], - interactive=True - ) - phaser_mix = gr.Slider( - minimum=0, - maximum=1, - step=0.01, - value=0.5, - label=translations["phaser_mix"], - info=translations["phaser_mix_info"], - interactive=True - ) - phaser_centre_frequency_hz = gr.Slider( - minimum=50, - maximum=5000, - step=10, - value=1000, - label=translations["phaser_centre_frequency_hz"], - info=translations["phaser_centre_frequency_hz_info"], - interactive=True - ) - phaser_feedback = gr.Slider( - minimum=-1, - maximum=1, - step=0.01, - value=0, - label=translations["phaser_feedback"], - info=translations["phaser_feedback_info"], - interactive=True - ) - with gr.Accordion( - translations["delay"], - open=True, - visible=False - ) as delay_accordion: - delay_seconds = gr.Slider( - minimum=0, - maximum=5, - step=0.01, - value=0.5, - label=translations["delay_seconds"], - info=translations["delay_seconds_info"], - interactive=True - ) - delay_feedback = gr.Slider( - minimum=0, - maximum=1, - step=0.01, - value=0.5, - label=translations["delay_feedback"], - info=translations["delay_feedback_info"], - interactive=True - ) - delay_mix = gr.Slider( - minimum=0, - maximum=1, - step=0.01, - value=0.5, - label=translations["delay_mix"], - info=translations["delay_mix_info"], - interactive=True - ) - with gr.Accordion( - translations["compressor"], - open=True, - visible=False - ) as compressor_accordion: - compressor_threshold = gr.Slider( - minimum=-60, - maximum=0, - step=1, - value=-20, - label=translations["compressor_threshold_db"], - info=translations["compressor_threshold_db_info"], - interactive=True - ) - compressor_ratio = gr.Slider( - minimum=1, - maximum=20, - step=0.1, - value=1, - label=translations["compressor_ratio"], - info=translations["compressor_ratio_info"], - interactive=True - ) - compressor_attack = gr.Slider( - minimum=0.1, - maximum=100, - step=0.1, - value=10, - label=translations["compressor_attack_ms"], - info=translations["compressor_attack_ms_info"], - interactive=True - ) - compressor_release = gr.Slider( - minimum=10, - maximum=1000, - step=1, - value=100, - label=translations["compressor_release_ms"], - info=translations["compressor_release_ms_info"], - interactive=True - ) - with gr.Accordion( - translations["limiter"], - open=True, - visible=False - ) as limiter_accordion: - limiter_threshold = gr.Slider( - minimum=-60, - maximum=0, - step=1, - value=-1, - label=translations["limiter_threshold_db"], - info=translations["limiter_threshold_db_info"], - interactive=True - ) - limiter_release_time = gr.Slider( - minimum=10, - maximum=1000, - step=1, - value=100, - label=translations["limiter_release_ms"], - info=translations["limiter_release_ms_info"], - interactive=True - ) - with gr.Row(): - distortion_gain = gr.Slider( - minimum=0, - maximum=50, - step=1, - value=20, - label=translations["distortion"], - info=translations["distortion_info"], - interactive=True, - visible=False - ) - pitch_shift_semitones = gr.Slider( - minimum=-24, - maximum=24, - step=1, - value=0, - label=translations["pitch"], - info=translations["pitch_info"], - interactive=True, - visible=False - ) - gain_db = gr.Slider( - minimum=-60, - maximum=60, - step=1, - value=0, - label=translations["gain"], - info=translations["gain_info"], - interactive=True, - visible=False - ) - bitcrush_bit_depth = gr.Slider( - minimum=1, - maximum=24, - step=1, - value=16, - label=translations["bitcrush_bit_depth"], - info=translations["bitcrush_bit_depth_info"], - interactive=True, - visible=False - ) - clipping_threshold = gr.Slider( - minimum=-60, - maximum=0, - step=1, - value=-1, - label=translations["clipping_threshold_db"], - info=translations["clipping_threshold_db_info"], - interactive=True, - visible=False - ) - with gr.Row(): - with gr.Accordion( - translations["use_presets"], - open=False - ): - with gr.Row(): - presets_name = gr.Dropdown( - label=translations["file_preset"], - choices=realtime_presets_file, - value=realtime_presets_file[0] if len(realtime_presets_file) > 0 else '', - interactive=True, - allow_custom_value=True - ) - with gr.Row(): - load_preset_button = gr.Button( - translations["load_file"], - variant="primary" - ) - refresh_preset_button = gr.Button( - translations["refresh"] - ) - with gr.Accordion( - translations["export_file"], - open=False - ): - with gr.Row(): - with gr.Column(): - name_to_save_file = gr.Textbox( - label=translations["filename_to_save"] - ) - save_file_button = gr.Button( - translations["export_file"] - ) - with gr.Row(): - upload_presets = gr.Files( - label=translations["upload_presets"], - file_types=[".realtime.json"] - ) - with gr.Row(): - json_audio_hidden = gr.JSON( - visible=False - ) - json_button_hidden = gr.JSON( - visible=False - ) - json_value_hidden = gr.JSON( - visible=False - ) - with gr.Row(): - post_process.change( - fn=lambda a: [ - visibleFalse(a) - for _ in range(11) - ], - inputs=[ - post_process - ], - outputs=[ - reverb, - chorus, - delay, - phaser, - compressor, - pitch_shift, - limiter, - distortion, - gain, - bitcrush, - clipping - ] - ) - reverb.change( - fn=visible, - inputs=[ - reverb - ], - outputs=[ - reverb_accordion - ] - ) - chorus.change( - fn=visible, - inputs=[ - chorus - ], - outputs=[ - chorus_accordion - ] - ) - with gr.Row(): - delay.change( - fn=visible, - inputs=[ - delay - ], - outputs=[ - delay_accordion - ] - ) - phaser.change( - fn=visible, - inputs=[ - phaser - ], - outputs=[ - phaser_accordion - ] - ) - compressor.change( - fn=visible, - inputs=[ - compressor - ], - outputs=[ - compressor_accordion - ] - ) - with gr.Row(): - limiter.change( - fn=visible, - inputs=[ - limiter - ], - outputs=[ - limiter_accordion - ] - ) - distortion.change( - fn=visible, - inputs=[ - distortion - ], - outputs=[ - distortion_gain - ] - ) - pitch_shift.change( - fn=visible, - inputs=[ - pitch_shift - ], - outputs=[ - pitch_shift_semitones - ] - ) - with gr.Row(): - gain.change( - fn=visible, - inputs=[ - gain - ], - outputs=[ - gain_db - ] - ) - bitcrush.change( - fn=visible, - inputs=[ - bitcrush - ], - outputs=[ - bitcrush_bit_depth - ] - ) - clipping.change( - fn=visible, - inputs=[ - clipping - ], - outputs=[ - clipping_threshold - ] - ) - with gr.Row(): - model_pth.change( - fn=get_index, - inputs=[ - model_pth - ], - outputs=[ - model_index - ] - ) - model_index.change( - fn=index_strength_show, - inputs=[ - model_index - ], - outputs=[ - index_strength, - nprobe - ] - ) - model_refresh.click( - fn=change_models_choices, - inputs=[], - outputs=[ - model_pth, - model_index - ] - ) - with gr.Row(): - unlock_full_method.change( - fn=lambda f0_method: { - "choices": [ - m for m in unlock_f0(f0_method)["choices"] - if m != "hybrid" - ], - "value": "rmvpe", - "__type__": "update" - }, - inputs=[ - unlock_full_method - ], - outputs=[ - f0_method - ] - ) - f0_method.change( - fn=lambda f0_method: hoplength_show(f0_method, None), - inputs=[ - f0_method - ], - outputs=[ - hop_length - ] - ) - embedder_mode.change( - fn=change_embedders_mode, - inputs=[ - embedder_mode - ], - outputs=[ - embedders - ] - ) - with gr.Row(): - embedders.change( - fn=lambda embedders: visible(embedders == "custom"), - inputs=[ - embedders - ], - outputs=[ - custom_embedders - ] - ) - f0_autotune.change( - fn=visible, - inputs=[ - f0_autotune - ], - outputs=[ - f0_autotune_strength - ] - ) - clean_audio.change( - fn=visible, - inputs=[ - clean_audio - ], - outputs=[ - clean_strength - ] - ) - with gr.Row(): - if not client_mode: - input_audio_device.change( - fn=update_audio_device, - inputs=[ - input_audio_device, - output_audio_device, - monitor_output_device, - monitor - ], - outputs=[ - monitor_output_device, - monitor_audio_gain, - asio_row, - input_asio_channels, - output_asio_channels, - monitor_asio_channels, - input_audio_sample_rate, - output_audio_sample_rate, - monitor_audio_sample_rate, - asio_output_stereo, - asio_monitor_stereo - ] - ) - output_audio_device.change( - fn=update_audio_device, - inputs=[ - input_audio_device, - output_audio_device, - monitor_output_device, - monitor - ], - outputs=[ - monitor_output_device, - monitor_audio_gain, - asio_row, - input_asio_channels, - output_asio_channels, - monitor_asio_channels, - input_audio_sample_rate, - output_audio_sample_rate, - monitor_audio_sample_rate, - asio_output_stereo, - asio_monitor_stereo - ] - ) - monitor_output_device.change( - fn=update_audio_device, - inputs=[ - input_audio_device, - output_audio_device, - monitor_output_device, - monitor - ], - outputs=[ - monitor_output_device, - monitor_audio_gain, - asio_row, - input_asio_channels, - output_asio_channels, - monitor_asio_channels, - input_audio_sample_rate, - output_audio_sample_rate, - monitor_audio_sample_rate, - asio_output_stereo, - asio_monitor_stereo - ] - ) - with gr.Row(): - embedders_mix.change( - fn=visible, - inputs=[ - embedders_mix - ], - outputs=[ - embedders_mix_column - ] - ) - if not client_mode: - monitor.change( - fn=update_audio_device, - inputs=[ - input_audio_device, - output_audio_device, - monitor_output_device, - monitor - ], - outputs=[ - monitor_output_device, - monitor_audio_gain, - asio_row, - input_asio_channels, - output_asio_channels, - monitor_asio_channels, - input_audio_sample_rate, - output_audio_sample_rate, - monitor_audio_sample_rate, - asio_output_stereo, - asio_monitor_stereo - ] - ) - with gr.Row(): - proposal_pitch.change( - fn=visible, - inputs=[ - proposal_pitch - ], - outputs=[ - proposal_pitch_threshold - ] - ) - vad_enabled.change( - fn=lambda a: [ - visible(a) - for _ in range(2) - ], - inputs=[ - vad_enabled - ], - outputs=[ - vad_sensitivity, - vad_frame_ms - ] - ) - if not client_mode: - refresh_audio_device.click( - fn=change_audio_device_choices, - inputs=[], - outputs=[ - input_audio_device, - output_audio_device, - monitor_output_device - ] - ) - else: - refresh_audio_device.click( - fn=None, - js="getAudioDevices", - inputs=[], - outputs=[ - json_audio_hidden - ] - ) - with gr.Row(): - model_pth.change( - fn=get_speakers_id_and_architecture, - inputs=[ - model_pth - ], - outputs=[ - sids, - noise_scale - ] - ) - if client_mode: - json_audio_hidden.change( - fn=update_dropdowns_from_json, - inputs=[ - json_audio_hidden - ], - outputs=[ - input_audio_device, - output_audio_device, - monitor_output_device - ] - ) - json_button_hidden.change( - fn=update_button_from_json, - inputs=[ - json_button_hidden - ], - outputs=[ - start_realtime, - stop_realtime - ] - ) - json_value_hidden.change( - fn=update_value_from_json, - inputs=[ - json_value_hidden - ], - outputs=[ - record_audio, - record_output - ] - ) - with gr.Row(): - if not client_mode: - start_realtime.click( - fn=realtime_start, - inputs=[ - monitor, - exclusive_mode, - vad_enabled, - input_audio_device, - output_audio_device, - monitor_output_device, - input_audio_gain, - output_audio_gain, - monitor_audio_gain, - input_asio_channels, - output_asio_channels, - monitor_asio_channels, - input_audio_sample_rate, - output_audio_sample_rate, - monitor_audio_sample_rate, - chunk_size, - pitch, - model_pth, - model_index, - index_strength, - predictor_onnx, - f0_method, - hop_length, - embedder_mode, - embedders, - custom_embedders, - f0_autotune, - proposal_pitch, - f0_autotune_strength, - proposal_pitch_threshold, - rms_mix_rate, - protect, - filter_radius, - silent_threshold, - extra_convert_size, - cross_fade_overlap_size, - vad_sensitivity, - vad_frame_ms, - clean_audio, - clean_strength, - post_process, - sids, - noise_scale, - embedders_mix, - embedders_mix_layers, - embedders_mix_ratio, - chorus, - distortion, - reverb, - pitch_shift, - delay, - compressor, - limiter, - gain, - bitcrush, - clipping, - phaser, - chorus_depth, - chorus_rate, - chorus_mix, - chorus_center_delay, - chorus_feedback, - distortion_gain, - reverb_room_size, - reverb_damping, - reverb_wet_gain, - reverb_dry_gain, - reverb_width, - reverb_freeze_mode, - pitch_shift_semitones, - delay_seconds, - delay_feedback, - delay_mix, - compressor_threshold, - compressor_ratio, - compressor_attack, - compressor_release, - limiter_threshold, - limiter_release_time, - gain_db, - bitcrush_bit_depth, - clipping_threshold, - phaser_rate_hz, - phaser_depth, - phaser_centre_frequency_hz, - phaser_feedback, - phaser_mix, - nprobe - ], - outputs=[ - status, - start_realtime, - stop_realtime - ] - ) - stop_realtime.click( - fn=realtime_stop, - inputs=[], - outputs=[ - status, - start_realtime, - stop_realtime - ] - ) - record_audio.click( - fn=soundfile_record_audio, - inputs=[ - record_audio, - record_audio_path, - export_format - ], - outputs=[ - record_audio, - record_output - ] - ) - else: - start_realtime.click( - fn=None, - js="StreamAudioRealtime", - inputs=[ - monitor, - vad_enabled, - input_audio_device, - output_audio_device, - monitor_output_device, - input_audio_gain, - output_audio_gain, - monitor_audio_gain, - chunk_size, - pitch, - model_pth, - model_index, - index_strength, - predictor_onnx, - f0_method, - hop_length, - embedder_mode, - embedders, - custom_embedders, - f0_autotune, - proposal_pitch, - f0_autotune_strength, - proposal_pitch_threshold, - rms_mix_rate, - protect, - filter_radius, - silent_threshold, - extra_convert_size, - cross_fade_overlap_size, - vad_sensitivity, - vad_frame_ms, - clean_audio, - clean_strength, - exclusive_mode, - post_process, - sids, - noise_scale, - embedders_mix, - embedders_mix_layers, - embedders_mix_ratio, - chorus, - distortion, - reverb, - pitch_shift, - delay, - compressor, - limiter, - gain, - bitcrush, - clipping, - phaser, - chorus_depth, - chorus_rate, - chorus_mix, - chorus_center_delay, - chorus_feedback, - distortion_gain, - reverb_room_size, - reverb_damping, - reverb_wet_gain, - reverb_dry_gain, - reverb_width, - reverb_freeze_mode, - pitch_shift_semitones, - delay_seconds, - delay_feedback, - delay_mix, - compressor_threshold, - compressor_ratio, - compressor_attack, - compressor_release, - limiter_threshold, - limiter_release_time, - gain_db, - bitcrush_bit_depth, - clipping_threshold, - phaser_rate_hz, - phaser_depth, - phaser_centre_frequency_hz, - phaser_feedback, - phaser_mix, - nprobe - ], - outputs=[ - json_button_hidden - ] - ) - stop_realtime.click( - fn=None, - js="StopAudioStream", - inputs=[], - outputs=[ - json_button_hidden - ] - ) - record_audio.click( - fn=None, - js="SoundfileRecordAudio", - inputs=[ - record_audio, - record_audio_path, - export_format - ], - outputs=[ - json_value_hidden - ] - ) - with gr.Row(): - cross_fade_overlap_size.change( - js="(value) => window.ChangeConfig(value, 'cross_fade_overlap_size')" if client_mode else None, - fn=(lambda value: change_config(value, "cross_fade_overlap_size")) if not client_mode else None, - inputs=[ - cross_fade_overlap_size - ], - outputs=[] - ) - extra_convert_size.change( - js="(value) => window.ChangeConfig(value, 'extra_convert_size')" if client_mode else None, - fn=(lambda value: change_config(value, "extra_convert_size")) if not client_mode else None, - inputs=[ - extra_convert_size - ], - outputs=[] - ) - silent_threshold.change( - js="(value) => window.ChangeConfig(value, 'silent_threshold')" if client_mode else None, - fn=(lambda value: change_config(value, "silent_threshold")) if not client_mode else None, - inputs=[ - silent_threshold - ], - outputs=[] - ) - with gr.Row(): - vad_enabled.change( - js="(value) => window.ChangeConfig(value, 'vad_enabled')" if client_mode else None, - fn=(lambda value: change_config(value, "vad_enabled")) if not client_mode else None, - inputs=[ - vad_enabled - ], - outputs=[] - ) - vad_sensitivity.change( - js="(value) => window.ChangeConfig(value, 'vad_sensitivity')" if client_mode else None, - fn=(lambda value: change_config(value, "vad_sensitivity")) if not client_mode else None, - inputs=[ - vad_sensitivity - ], - outputs=[] - ) - vad_frame_ms.change( - js="(value) => window.ChangeConfig(value, 'vad_frame_ms')" if client_mode else None, - fn=(lambda value: change_config(value, "vad_frame_ms")) if not client_mode else None, - inputs=[ - vad_frame_ms - ], - outputs=[] - ) - with gr.Row(): - input_audio_gain.change( - js="(value) => window.ChangeConfig(value, 'input_audio_gain')" if client_mode else None, - fn=(lambda value: change_config(value / 100.0, "input_audio_gain")) if not client_mode else None, - inputs=[ - input_audio_gain - ], - outputs=[] - ) - output_audio_gain.change( - js="(value) => window.ChangeConfig(value, 'output_audio_gain')" if client_mode else None, - fn=(lambda value: change_config(value / 100.0, "output_audio_gain")) if not client_mode else None, - inputs=[ - output_audio_gain - ], - outputs=[] - ) - monitor_audio_gain.change( - js="(value) => window.ChangeConfig(value, 'monitor_audio_gain')" if client_mode else None, - fn=(lambda value: change_config(value / 100.0, "monitor_audio_gain")) if not client_mode else None, - inputs=[ - monitor_audio_gain - ], - outputs=[] - ) - with gr.Row(): - index_strength.change( - js="(value) => window.ChangeConfig(value, 'index_rate')" if client_mode else None, - fn=(lambda value: change_config(value, "index_rate")) if not client_mode else None, - inputs=[ - index_strength - ], - outputs=[] - ) - nprobe.change( - js="(value) => window.ChangeConfig(value, 'nprobe')" if client_mode else None, - fn=(lambda value: change_config(value, "nprobe")) if not client_mode else None, - inputs=[ - nprobe - ], - outputs=[] - ) - rms_mix_rate.change( - js="(value) => window.ChangeConfig(value, 'rms_mix_rate')" if client_mode else None, - fn=(lambda value: change_config(value, "rms_mix_rate")) if not client_mode else None, - inputs=[ - rms_mix_rate - ], - outputs=[] - ) - protect.change( - js="(value) => window.ChangeConfig(value, 'protect')" if client_mode else None, - fn=(lambda value: change_config(value, "protect")) if not client_mode else None, - inputs=[ - protect - ], - outputs=[] - ) - with gr.Row(): - pitch.change( - js="(value) => window.ChangeConfig(value, 'f0_up_key')" if client_mode else None, - fn=(lambda value: change_config(value, "f0_up_key")) if not client_mode else None, - inputs=[ - pitch - ], - outputs=[] - ) - f0_autotune.change( - js="(value) => window.ChangeConfig(value, 'f0_autotune')" if client_mode else None, - fn=(lambda value: change_config(value, "f0_autotune")) if not client_mode else None, - inputs=[ - f0_autotune - ], - outputs=[] - ) - f0_autotune_strength.change( - js="(value) => window.ChangeConfig(value, 'f0_autotune_strength')" if client_mode else None, - fn=(lambda value: change_config(value, "f0_autotune_strength")) if not client_mode else None, - inputs=[ - f0_autotune_strength - ], - outputs=[] - ) - with gr.Row(): - proposal_pitch.change( - js="(value) => window.ChangeConfig(value, 'proposal_pitch')" if client_mode else None, - fn=(lambda value: change_config(value, "proposal_pitch")) if not client_mode else None, - inputs=[ - proposal_pitch - ], - outputs=[] - ) - proposal_pitch_threshold.change( - js="(value) => window.ChangeConfig(value, 'proposal_pitch_threshold')" if client_mode else None, - fn=(lambda value: change_config(value, "proposal_pitch_threshold")) if not client_mode else None, - inputs=[ - proposal_pitch_threshold - ], - outputs=[] - ) - clean_audio.change( - js="(value) => window.ChangeConfig(value, 'clean_audio')" if client_mode else None, - fn=(lambda value: change_config(value, "clean_audio")) if not client_mode else None, - inputs=[ - clean_audio - ], - outputs=[] - ) - with gr.Row(): - clean_strength.change( - js="(value) => window.ChangeConfig(value, 'clean_strength')" if client_mode else None, - fn=(lambda value: change_config(value, "clean_strength")) if not client_mode else None, - inputs=[ - clean_strength - ], - outputs=[] - ) - post_process.change( - js="(value) => window.ChangeConfig(value, 'post_process')" if client_mode else None, - fn=(lambda value: change_config(value, "post_process")) if not client_mode else None, - inputs=[post_process], - outputs=[] - ) - reverb.change( - js="(value) => window.ChangeConfig(value, 'reverb', if_kwargs=true)" if client_mode else None, - fn=(lambda value: change_config(value, "reverb", if_kwargs=True)) if not client_mode else None, - inputs=[ - reverb - ], - outputs=[] - ) - with gr.Row(): - chorus.change( - js="(value) => window.ChangeConfig(value, 'chorus', if_kwargs=true)" if client_mode else None, - fn=(lambda value: change_config(value, "chorus", if_kwargs=True)) if not client_mode else None, - inputs=[ - chorus - ], - outputs=[] - ) - delay.change( - js="(value) => window.ChangeConfig(value, 'delay', if_kwargs=true)" if client_mode else None, - fn=(lambda value: change_config(value, "delay", if_kwargs=True)) if not client_mode else None, - inputs=[ - delay - ], - outputs=[] - ) - phaser.change( - js="(value) => window.ChangeConfig(value, 'phaser', if_kwargs=true)" if client_mode else None, - fn=(lambda value: change_config(value, "phaser", if_kwargs=True)) if not client_mode else None, - inputs=[ - phaser - ], - outputs=[] - ) - with gr.Row(): - compressor.change( - js="(value) => window.ChangeConfig(value, 'compressor', if_kwargs=true)" if client_mode else None, - fn=(lambda value: change_config(value, "compressor", if_kwargs=True)) if not client_mode else None, - inputs=[ - compressor - ], - outputs=[] - ) - limiter.change( - js="(value) => window.ChangeConfig(value, 'limiter', if_kwargs=true)" if client_mode else None, - fn=(lambda value: change_config(value, "limiter", if_kwargs=True)) if not client_mode else None, - inputs=[ - limiter - ], - outputs=[] - ) - distortion.change( - js="(value) => window.ChangeConfig(value, 'distortion', if_kwargs=true)" if client_mode else None, - fn=(lambda value: change_config(value, "distortion", if_kwargs=True)) if not client_mode else None, - inputs=[ - distortion - ], - outputs=[] - ) - with gr.Row(): - pitch_shift.change( - js="(value) => window.ChangeConfig(value, 'pitch_shift', if_kwargs=true)" if client_mode else None, - fn=(lambda value: change_config(value, "pitch_shift", if_kwargs=True)) if not client_mode else None, - inputs=[ - pitch_shift - ], - outputs=[] - ) - gain.change( - js="(value) => window.ChangeConfig(value, 'gain', if_kwargs=true)" if client_mode else None, - fn=(lambda value: change_config(value, "gain", if_kwargs=True)) if not client_mode else None, - inputs=[ - gain - ], - outputs=[] - ) - bitcrush.change( - js="(value) => window.ChangeConfig(value, 'bitcrush', if_kwargs=true)" if client_mode else None, - fn=(lambda value: change_config(value, "bitcrush", if_kwargs=True)) if not client_mode else None, - inputs=[ - bitcrush - ], - outputs=[] - ) - with gr.Row(): - clipping.change( - js="(value) => window.ChangeConfig(value, 'clipping', if_kwargs=true)" if client_mode else None, - fn=(lambda value: change_config(value, "clipping", if_kwargs=True)) if not client_mode else None, - inputs=[ - clipping - ], - outputs=[] - ) - pitch_shift_semitones.change( - js="(value) => window.ChangeConfig(value, 'pitch_shift_semitones', if_kwargs=true)" if client_mode else None, - fn=(lambda value: change_config(value, "pitch_shift_semitones", if_kwargs=True)) if not client_mode else None, - inputs=[ - pitch_shift_semitones - ], - outputs=[] - ) - gain_db.change( - js="(value) => window.ChangeConfig(value, 'gain_db', if_kwargs=true)" if client_mode else None, - fn=(lambda value: change_config(value, "gain_db", if_kwargs=True)) if not client_mode else None, - inputs=[ - gain_db - ], - outputs=[] - ) - with gr.Row(): - phaser_depth.change( - js="(value) => window.ChangeConfig(value, 'phaser_depth', if_kwargs=true)" if client_mode else None, - fn=(lambda value: change_config(value, "phaser_depth", if_kwargs=True)) if not client_mode else None, - inputs=[ - phaser_depth - ], - outputs=[] - ) - phaser_rate_hz.change( - js="(value) => window.ChangeConfig(value, 'phaser_rate_hz', if_kwargs=true)" if client_mode else None, - fn=(lambda value: change_config(value, "phaser_rate_hz", if_kwargs=True)) if not client_mode else None, - inputs=[ - phaser_rate_hz - ], - outputs=[] - ) - phaser_mix.change( - js="(value) => window.ChangeConfig(value, 'phaser_mix', if_kwargs=true)" if client_mode else None, - fn=(lambda value: change_config(value, "phaser_mix", if_kwargs=True)) if not client_mode else None, - inputs=[ - phaser_mix - ], - outputs=[] - ) - with gr.Row(): - phaser_centre_frequency_hz.change( - js="(value) => window.ChangeConfig(value, 'phaser_centre_frequency_hz', if_kwargs=true)" if client_mode else None, - fn=(lambda value: change_config(value, "phaser_centre_frequency_hz", if_kwargs=True)) if not client_mode else None, - inputs=[phaser_centre_frequency_hz], - outputs=[] - ) - phaser_feedback.change( - js="(value) => window.ChangeConfig(value, 'phaser_feedback', if_kwargs=true)" if client_mode else None, - fn=(lambda value: change_config(value, "phaser_feedback", if_kwargs=True)) if not client_mode else None, - inputs=[ - phaser_feedback - ], - outputs=[] - ) - reverb_room_size.change( - js="(value) => window.ChangeConfig(value, 'reverb_room_size', if_kwargs=true)" if client_mode else None, - fn=(lambda value: change_config(value, "reverb_room_size", if_kwargs=True)) if not client_mode else None, - inputs=[ - reverb_room_size - ], - outputs=[] - ) - with gr.Row(): - reverb_damping.change( - js="(value) => window.ChangeConfig(value, 'reverb_damping', if_kwargs=true)" if client_mode else None, - fn=(lambda value: change_config(value, "reverb_damping", if_kwargs=True)) if not client_mode else None, - inputs=[ - reverb_damping - ], - outputs=[] - ) - reverb_wet_gain.change( - js="(value) => window.ChangeConfig(value, 'reverb_wet_level', if_kwargs=true)" if client_mode else None, - fn=(lambda value: change_config(value, "reverb_wet_level", if_kwargs=True)) if not client_mode else None, - inputs=[ - reverb_wet_gain - ], - outputs=[] - ) - reverb_dry_gain.change( - js="(value) => window.ChangeConfig(value, 'reverb_dry_level', if_kwargs=true)" if client_mode else None, - fn=(lambda value: change_config(value, "reverb_dry_level", if_kwargs=True)) if not client_mode else None, - inputs=[ - reverb_dry_gain - ], - outputs=[] - ) - with gr.Row(): - reverb_width.change( - js="(value) => window.ChangeConfig(value, 'reverb_width', if_kwargs=true)" if client_mode else None, - fn=(lambda value: change_config(value, "reverb_width", if_kwargs=True)) if not client_mode else None, - inputs=[ - reverb_width - ], - outputs=[] - ) - reverb_freeze_mode.change( - js="(value) => window.ChangeConfig(value, 'reverb_freeze_mode', if_kwargs=true)" if client_mode else None, - fn=(lambda value: change_config(value, "reverb_freeze_mode", if_kwargs=True)) if not client_mode else None, - inputs=[ - reverb_freeze_mode - ], - outputs=[] - ) - limiter_threshold.change( - js="(value) => window.ChangeConfig(value, 'limiter_threshold', if_kwargs=true)" if client_mode else None, - fn=(lambda value: change_config(value, "limiter_threshold", if_kwargs=True)) if not client_mode else None, - inputs=[ - limiter_threshold - ], - outputs=[] - ) - with gr.Row(): - limiter_release_time.change( - js="(value) => window.ChangeConfig(value, 'limiter_release', if_kwargs=true)" if client_mode else None, - fn=(lambda value: change_config(value, "limiter_release", if_kwargs=True)) if not client_mode else None, - inputs=[ - limiter_release_time - ], - outputs=[] - ) - chorus_rate.change( - js="(value) => window.ChangeConfig(value, 'chorus_rate', if_kwargs=true)" if client_mode else None, - fn=(lambda value: change_config(value, "chorus_rate", if_kwargs=True)) if not client_mode else None, - inputs=[ - chorus_rate - ], - outputs=[] - ) - chorus_depth.change( - js="(value) => window.ChangeConfig(value, 'chorus_depth', if_kwargs=true)" if client_mode else None, - fn=(lambda value: change_config(value, "chorus_depth", if_kwargs=True)) if not client_mode else None, - inputs=[ - chorus_depth - ], - outputs=[] - ) - with gr.Row(): - chorus_center_delay.change( - js="(value) => window.ChangeConfig(value, 'chorus_delay', if_kwargs=true)" if client_mode else None, - fn=(lambda value: change_config(value, "chorus_delay", if_kwargs=True)) if not client_mode else None, - inputs=[ - chorus_center_delay - ], - outputs=[] - ) - chorus_feedback.change( - js="(value) => window.ChangeConfig(value, 'chorus_feedback', if_kwargs=true)" if client_mode else None, - fn=(lambda value: change_config(value, "chorus_feedback", if_kwargs=True)) if not client_mode else None, - inputs=[ - chorus_feedback - ], - outputs=[] - ) - chorus_mix.change( - js="(value) => window.ChangeConfig(value, 'chorus_mix', if_kwargs=true)" if client_mode else None, - fn=(lambda value: change_config(value, "chorus_mix", if_kwargs=True)) if not client_mode else None, - inputs=[ - chorus_mix - ], - outputs=[] - ) - with gr.Row(): - bitcrush_bit_depth.change( - js="(value) => window.ChangeConfig(value, 'bitcrush_bit_depth', if_kwargs=true)" if client_mode else None, - fn=(lambda value: change_config(value, "bitcrush_bit_depth", if_kwargs=True)) if not client_mode else None, - inputs=[ - bitcrush_bit_depth - ], - outputs=[] - ) - clipping_threshold.change( - js="(value) => window.ChangeConfig(value, 'clipping_threshold', if_kwargs=true)" if client_mode else None, - fn=(lambda value: change_config(value, "clipping_threshold", if_kwargs=True)) if not client_mode else None, - inputs=[ - clipping_threshold - ], - outputs=[] - ) - distortion_gain.change( - js="(value) => window.ChangeConfig(value, 'distortion_gain', if_kwargs=true)" if client_mode else None, - fn=(lambda value: change_config(value, "distortion_gain", if_kwargs=True)) if not client_mode else None, - inputs=[ - distortion_gain - ], - outputs=[] - ) - with gr.Row(): - compressor_threshold.change( - js="(value) => window.ChangeConfig(value, 'compressor_threshold', if_kwargs=true)" if client_mode else None, - fn=(lambda value: change_config(value, "compressor_threshold", if_kwargs=True)) if not client_mode else None, - inputs=[ - compressor_threshold - ], - outputs=[] - ) - compressor_ratio.change( - js="(value) => window.ChangeConfig(value, 'compressor_ratio', if_kwargs=true)" if client_mode else None, - fn=(lambda value: change_config(value, "compressor_ratio", if_kwargs=True)) if not client_mode else None, - inputs=[ - compressor_ratio - ], - outputs=[] - ) - compressor_attack.change( - js="(value) => window.ChangeConfig(value, 'compressor_attack', if_kwargs=true)" if client_mode else None, - fn=(lambda value: change_config(value, "compressor_attack", if_kwargs=True)) if not client_mode else None, - inputs=[ - compressor_attack - ], - outputs=[] - ) - with gr.Row(): - compressor_release.change( - js="(value) => window.ChangeConfig(value, 'compressor_release', if_kwargs=true)" if client_mode else None, - fn=(lambda value: change_config(value, "compressor_release", if_kwargs=True)) if not client_mode else None, - inputs=[ - compressor_release - ], - outputs=[] - ) - delay_seconds.change( - js="(value) => window.ChangeConfig(value, 'delay_seconds', if_kwargs=true)" if client_mode else None, - fn=(lambda value: change_config(value, "delay_seconds", if_kwargs=True)) if not client_mode else None, - inputs=[ - delay_seconds - ], - outputs=[] - ) - delay_feedback.change( - js="(value) => window.ChangeConfig(value, 'delay_feedback', if_kwargs=true)" if client_mode else None, - fn=(lambda value: change_config(value, "delay_feedback", if_kwargs=True)) if not client_mode else None, - inputs=[ - delay_feedback - ], - outputs=[] - ) - with gr.Row(): - delay_mix.change( - js="(value) => window.ChangeConfig(value, 'delay_mix', if_kwargs=true)" if client_mode else None, - fn=(lambda value: change_config(value, "delay_mix", if_kwargs=True)) if not client_mode else None, - inputs=[ - delay_mix - ], - outputs=[] - ) - model_pth.change( - js="(value) => window.ChangeConfig(value, 'model_path')" if client_mode else None, - fn=(lambda value: change_config(value, "model_path")) if not client_mode else None, - inputs=[ - model_pth - ], - outputs=[] - ) - model_index.change( - js="(value) => window.ChangeConfig(value, 'index_path')" if client_mode else None, - fn=(lambda value: change_config(value, "index_path")) if not client_mode else None, - inputs=[ - model_index - ], - outputs=[] - ) - with gr.Row(): - f0_method.change( - js="(value) => window.ChangeConfig(value, 'f0_method')" if client_mode else None, - fn=(lambda value: change_config(value, "f0_method")) if not client_mode else None, - inputs=[ - f0_method - ], - outputs=[] - ) - predictor_onnx.change( - js="(value) => window.ChangeConfig(value, 'predictor_onnx')" if client_mode else None, - fn=(lambda value: change_config(value, "predictor_onnx")) if not client_mode else None, - inputs=[ - predictor_onnx - ], - outputs=[] - ) - embedders.change( - js="(value) => window.ChangeConfig(value, 'embedder_model')" if client_mode else None, - fn=(lambda value: change_config(value, "embedder_model")) if not client_mode else None, - inputs=[ - embedders - ], - outputs=[] - ) - with gr.Row(): - custom_embedders.change( - js="(value) => window.ChangeConfig(value, 'embedder_model_custom')" if client_mode else None, - fn=(lambda value: change_config(value, "embedder_model_custom")) if not client_mode else None, - inputs=[ - custom_embedders - ], - outputs=[] - ) - embedder_mode.change( - js="(value) => window.ChangeConfig(value, 'embedders_mode')" if client_mode else None, - fn=(lambda value: change_config(value, "embedders_mode")) if not client_mode else None, - inputs=[ - embedder_mode - ], - outputs=[] - ) - with gr.Row(): - embedders_mix.change( - js="(value) => window.ChangeConfig(value, 'embedders_mix')" if client_mode else None, - fn=(lambda value: change_config(value, "embedders_mix")) if not client_mode else None, - inputs=[ - embedders_mix - ], - outputs=[] - ) - embedders_mix_layers.change( - js="(value) => window.ChangeConfig(value, 'embedders_mix_layers')" if client_mode else None, - fn=(lambda value: change_config(value, "embedders_mix_layers")) if not client_mode else None, - inputs=[ - embedders_mix_layers - ], - outputs=[] - ) - embedders_mix_ratio.change( - js="(value) => window.ChangeConfig(value, 'embedders_mix_ratio')" if client_mode else None, - fn=(lambda value: change_config(value, "embedders_mix_ratio")) if not client_mode else None, - inputs=[ - embedders_mix_ratio - ], - outputs=[] - ) - with gr.Row(): - noise_scale.change( - js="(value) => window.ChangeConfig(value, 'noise_scale')" if client_mode else None, - fn=(lambda value: change_config(value, "noise_scale")) if not client_mode else None, - inputs=[ - noise_scale - ], - outputs=[] - ) - with gr.Row(): - load_preset_button.click( - fn=load_realtime_config, - inputs=[ - presets_name, - exclusive_mode, - vad_enabled, - chunk_size, - pitch, - index_strength, - predictor_onnx, - hop_length, - embedder_mode, - f0_autotune, - proposal_pitch, - f0_autotune_strength, - proposal_pitch_threshold, - rms_mix_rate, - protect, - filter_radius, - silent_threshold, - extra_convert_size, - cross_fade_overlap_size, - vad_sensitivity, - vad_frame_ms, - clean_audio, - clean_strength, - post_process, - embedders_mix, - embedders_mix_layers, - embedders_mix_ratio, - chorus, - distortion, - reverb, - pitch_shift, - delay, - compressor, - limiter, - gain, - bitcrush, - clipping, - phaser, - chorus_depth, - chorus_rate, - chorus_mix, - chorus_center_delay, - chorus_feedback, - distortion_gain, - reverb_room_size, - reverb_damping, - reverb_wet_gain, - reverb_dry_gain, - reverb_width, - reverb_freeze_mode, - pitch_shift_semitones, - delay_seconds, - delay_feedback, - delay_mix, - compressor_threshold, - compressor_ratio, - compressor_attack, - compressor_release, - limiter_threshold, - limiter_release_time, - gain_db, - bitcrush_bit_depth, - clipping_threshold, - phaser_rate_hz, - phaser_depth, - phaser_centre_frequency_hz, - phaser_feedback, - phaser_mix - ], - outputs=[ - exclusive_mode, - vad_enabled, - chunk_size, - pitch, - index_strength, - predictor_onnx, - hop_length, - embedder_mode, - f0_autotune, - proposal_pitch, - f0_autotune_strength, - proposal_pitch_threshold, - rms_mix_rate, - protect, - filter_radius, - silent_threshold, - extra_convert_size, - cross_fade_overlap_size, - vad_sensitivity, - vad_frame_ms, - clean_audio, - clean_strength, - post_process, - embedders_mix, - embedders_mix_layers, - embedders_mix_ratio, - chorus, - distortion, - reverb, - pitch_shift, - delay, - compressor, - limiter, - gain, - bitcrush, - clipping, - phaser, - chorus_depth, - chorus_rate, - chorus_mix, - chorus_center_delay, - chorus_feedback, - distortion_gain, - reverb_room_size, - reverb_damping, - reverb_wet_gain, - reverb_dry_gain, - reverb_width, - reverb_freeze_mode, - pitch_shift_semitones, - delay_seconds, - delay_feedback, - delay_mix, - compressor_threshold, - compressor_ratio, - compressor_attack, - compressor_release, - limiter_threshold, - limiter_release_time, - gain_db, - bitcrush_bit_depth, - clipping_threshold, - phaser_rate_hz, - phaser_depth, - phaser_centre_frequency_hz, - phaser_feedback, - phaser_mix - ] - ) - refresh_preset_button.click( - fn=change_realtime_preset_choices, - inputs=[], - outputs=[ - presets_name - ] - ) - save_file_button.click( - fn=save_realtime_config, - inputs=[ - name_to_save_file, - exclusive_mode, - vad_enabled, - chunk_size, - pitch, - index_strength, - predictor_onnx, - hop_length, - embedder_mode, - f0_autotune, - proposal_pitch, - f0_autotune_strength, - proposal_pitch_threshold, - rms_mix_rate, - protect, - filter_radius, - silent_threshold, - extra_convert_size, - cross_fade_overlap_size, - vad_sensitivity, - vad_frame_ms, - clean_audio, - clean_strength, - post_process, - embedders_mix, - embedders_mix_layers, - embedders_mix_ratio, - chorus, - distortion, - reverb, - pitch_shift, - delay, - compressor, - limiter, - gain, - bitcrush, - clipping, - phaser, - chorus_depth, - chorus_rate, - chorus_mix, - chorus_center_delay, - chorus_feedback, - distortion_gain, - reverb_room_size, - reverb_damping, - reverb_wet_gain, - reverb_dry_gain, - reverb_width, - reverb_freeze_mode, - pitch_shift_semitones, - delay_seconds, - delay_feedback, - delay_mix, - compressor_threshold, - compressor_ratio, - compressor_attack, - compressor_release, - limiter_threshold, - limiter_release_time, - gain_db, - bitcrush_bit_depth, - clipping_threshold, - phaser_rate_hz, - phaser_depth, - phaser_centre_frequency_hz, - phaser_feedback, - phaser_mix - ], - outputs=[ - presets_name - ] - ) - with gr.Row(): - upload_presets.upload( - fn=lambda presets_in: [ - shutil_move(preset.name, configs["presets_path"]) - for preset in presets_in - ][0], - inputs=[ - upload_presets - ], - outputs=[ - presets_name - ] - ) \ No newline at end of file diff --git a/main/app/tabs/training/child/create_dataset.py b/main/app/tabs/training/child/create_dataset.py deleted file mode 100644 index 8ff1c225eb4c44c3e7ef3def556b4687e10e732f..0000000000000000000000000000000000000000 --- a/main/app/tabs/training/child/create_dataset.py +++ /dev/null @@ -1,519 +0,0 @@ -import os -import sys - -import gradio as gr - -sys.path.append(os.getcwd()) - -from main.app.core.training import create_dataset - -from main.app.core.ui import ( - visible, - create_dataset_change, - valueFalse_interactive -) -from main.app.variables import ( - vr_models, - uvr_model, - mdx_models, - translations, - reverb_models, - denoise_models, - sample_rate_choice -) - -def create_dataset_tab(): - with gr.Row(): - gr.Markdown(translations["create_dataset_markdown_2"]) - with gr.Group(): - with gr.Row(): - separate = gr.Checkbox( - label=translations["separator_tab"], - value=False, - interactive=True - ) - clean_dataset = gr.Checkbox( - label=translations["clear_audio"], - value=False, - interactive=True - ) - skip_seconds = gr.Checkbox( - label=translations["skip"], - value=False, - interactive=True - ) - separate_reverb = gr.Checkbox( - label=translations["dereveb_audio"], - value=False, - interactive=False - ) - with gr.Row(visible=False) as row: - enable_tta = gr.Checkbox( - label=translations["enable_tta"], - value=False, - interactive=False - ) - high_end_process = gr.Checkbox( - label=translations["high_end_process"], - value=False, - interactive=False - ) - enable_post_process = gr.Checkbox( - label=translations["enable_post_process"], - value=False, - interactive=False - ) - enable_denoise = gr.Checkbox( - label=translations["denoise_mdx"], - value=False, - interactive=False - ) - with gr.Row(): - dataset_url = gr.Textbox( - label=translations["url_audio"], - info=translations["create_dataset_url"], - value="", - placeholder="https://www.youtube.com/...", - interactive=True, - scale=5 - ) - output_dataset = gr.Textbox( - label=translations["output_data"], - info=translations["output_data_info"], - value="dataset", - placeholder="dataset", - interactive=True - ) - with gr.Row(): - create_dataset_button = gr.Button( - translations["createdataset"], - variant="primary", - scale=2, - min_width=4000 - ) - with gr.Row(visible=False) as row_2: - model_name = gr.Dropdown( - label=translations["separator_model"], - value=uvr_model[0], - choices=uvr_model, - interactive=True - ) - reverb_model = gr.Dropdown( - label=translations["dereveb_model"], - value=list(reverb_models.keys())[0], - choices=list(reverb_models.keys()), - interactive=True - ) - denoise_model = gr.Dropdown( - label=translations["denoise_model"], - value=list(denoise_models.keys())[0], - choices=list(denoise_models.keys()), - interactive=True, - visible=False - ) - with gr.Row(): - with gr.Column(visible=False) as row_3: - with gr.Group(): - with gr.Row(): - overlap = gr.Radio( - label=translations["overlap"], - info=translations["overlap_info"], - choices=[ - "0.25", - "0.5", - "0.75", - "0.99" - ], - value="0.25", - interactive=True) - with gr.Row(): - window_size = gr.Slider( - label=translations["window_size"], - info=translations["window_size_info"], - minimum=320, - maximum=1024, - value=512, - step=32, - interactive=True, - visible=False - ) - with gr.Row(): - shifts = gr.Slider( - label=translations["shift"], - info=translations["shift_info"], - minimum=1, - maximum=20, - value=2, - step=1, - interactive=True - ) - segments_size = gr.Slider( - label=translations["segments_size"], - info=translations["segments_size_info"], - minimum=32, - maximum=3072, - value=256, - step=32, - interactive=True - ) - with gr.Row(): - batch_size = gr.Slider( - label=translations["batch_size"], - info=translations["mdx_batch_size_info"], - minimum=1, - maximum=64, - value=1, - step=1, - interactive=True, - visible=False - ) - hop_length = gr.Slider( - label=translations['hop_length'], - info=translations["hop_length_info"], - minimum=64, - maximum=8192, - value=1024, - step=1, - interactive=True, - visible=False - ) - with gr.Row(): - post_process_threshold = gr.Slider( - label=translations['post_process_threshold'], - info=translations["post_process_threshold_info"], - minimum=0.1, - maximum=0.3, - value=0.2, - step=0.1, - interactive=True, - visible=False - ) - aggression = gr.Slider( - label=translations['aggression'], - info=translations["aggression_info"], - minimum=1, - maximum=50, - value=5, - step=1, - interactive=True, - visible=False - ) - with gr.Column(): - sample_rate = gr.Radio( - choices=sample_rate_choice, - value=48000, - label=translations["sr"], - info=translations["sr_info"], - interactive=True - ) - clean_strength = gr.Slider( - minimum=0, - maximum=1, - step=0.1, - value=0.5, - label=translations["clean_strength"], - info=translations["clean_strength_info"], - interactive=True, - visible=False - ) - with gr.Row(): - skip_start = gr.Textbox( - label=translations["skip_start"], - info=translations["skip_start_info"], - value="", - placeholder="0,...", - interactive=True, - visible=False - ) - skip_end = gr.Textbox( - label=translations["skip_end"], - info=translations["skip_end_info"], - value="", - placeholder="0,...", - interactive=True, - visible=False - ) - create_dataset_info = gr.Textbox( - label=translations["create_dataset_info"], - value="", - interactive=False, - lines=2 - ) - with gr.Row(): - separate.change( - fn=lambda a: [ - visible(a) - for _ in range(3) - ], - inputs=[ - separate - ], - outputs=[ - row, - row_2, - row_3 - ] - ) - separate.change( - fn=valueFalse_interactive, - inputs=[ - separate - ], - outputs=[ - separate_reverb - ] - ) - separate.change( - fn=create_dataset_change, - inputs=[ - model_name, - reverb_model, - enable_post_process, - separate_reverb, - enable_denoise - ], - outputs=[ - reverb_model, - overlap, - segments_size, - hop_length, - batch_size, - shifts, - window_size, - aggression, - post_process_threshold, - denoise_model, - enable_tta, - high_end_process, - enable_post_process, - ] - ) - with gr.Row(): - model_name.change( - fn=create_dataset_change, - inputs=[ - model_name, - reverb_model, - enable_post_process, - separate_reverb, - enable_denoise - ], - outputs=[ - reverb_model, - overlap, - segments_size, - hop_length, - batch_size, - shifts, - window_size, - aggression, - post_process_threshold, - denoise_model, - enable_tta, - high_end_process, - enable_post_process, - ] - ) - reverb_model.change( - fn=create_dataset_change, - inputs=[ - model_name, - reverb_model, - enable_post_process, - separate_reverb, - enable_denoise - ], - outputs=[ - reverb_model, - overlap, - segments_size, - hop_length, - batch_size, - shifts, - window_size, - aggression, - post_process_threshold, - denoise_model, - enable_tta, - high_end_process, - enable_post_process, - ] - ) - denoise_model.change( - fn=create_dataset_change, - inputs=[ - model_name, - reverb_model, - enable_post_process, - separate_reverb, - enable_denoise - ], - outputs=[ - reverb_model, - overlap, - segments_size, - hop_length, - batch_size, - shifts, - window_size, - aggression, - post_process_threshold, - denoise_model, - enable_tta, - high_end_process, - enable_post_process, - ] - ) - with gr.Row(): - separate_reverb.change( - fn=create_dataset_change, - inputs=[ - model_name, - reverb_model, - enable_post_process, - separate_reverb, - enable_denoise - ], - outputs=[ - reverb_model, - overlap, - segments_size, - hop_length, - batch_size, - shifts, - window_size, - aggression, - post_process_threshold, - denoise_model, - enable_tta, - high_end_process, - enable_post_process, - ] - ) - enable_denoise.change( - fn=create_dataset_change, - inputs=[ - model_name, - reverb_model, - enable_post_process, - separate_reverb, - enable_denoise - ], - outputs=[ - reverb_model, - overlap, - segments_size, - hop_length, - batch_size, - shifts, - window_size, - aggression, - post_process_threshold, - denoise_model, - enable_tta, - high_end_process, - enable_post_process, - ] - ) - enable_post_process.change( - fn=create_dataset_change, - inputs=[ - model_name, - reverb_model, - enable_post_process, - separate_reverb, - enable_denoise - ], - outputs=[ - reverb_model, - overlap, - segments_size, - hop_length, - batch_size, - shifts, - window_size, - aggression, - post_process_threshold, - denoise_model, - enable_tta, - high_end_process, - enable_post_process, - ] - ) - with gr.Row(): - skip_seconds.change( - fn=lambda a: [ - visible(a) - for _ in range(2) - ], - inputs=[ - skip_seconds - ], - outputs=[ - skip_start, - skip_end - ] - ) - clean_dataset.change( - fn=visible, - inputs=[ - clean_dataset - ], - outputs=[ - clean_strength - ] - ) - with gr.Row(): - model_name.change( - fn=lambda a: valueFalse_interactive(a in list(mdx_models.keys()) + list(vr_models.keys())), - inputs=[ - model_name - ], - outputs=[ - enable_denoise - ] - ) - separate_reverb.change( - fn=valueFalse_interactive, - inputs=[ - separate_reverb - ], - outputs=[ - enable_denoise - ] - ) - with gr.Row(): - create_dataset_button.click( - fn=create_dataset, - inputs=[ - dataset_url, - output_dataset, - skip_seconds, - skip_start, - skip_end, - separate, - model_name, - reverb_model, - denoise_model, - sample_rate, - shifts, - batch_size, - overlap, - aggression, - hop_length, - window_size, - segments_size, - post_process_threshold, - enable_tta, - enable_denoise, - high_end_process, - enable_post_process, - separate_reverb, - clean_dataset, - clean_strength - ], - outputs=[ - create_dataset_info - ], - api_name="create_dataset" - ) \ No newline at end of file diff --git a/main/app/tabs/training/child/create_reference.py b/main/app/tabs/training/child/create_reference.py deleted file mode 100644 index 8fed8277a97b87de7fb0355cef5a55c1007bf89c..0000000000000000000000000000000000000000 --- a/main/app/tabs/training/child/create_reference.py +++ /dev/null @@ -1,390 +0,0 @@ -import os -import sys - -import gradio as gr - -sys.path.append(os.getcwd()) - -from main.app.core.training import create_reference - -from main.app.core.ui import ( - visible, - unlock_f0, - shutil_move, - change_audios_choices, - change_embedders_mode -) - -from main.app.variables import ( - configs, - method_f0, - file_types, - audio_params, - translations, - embedders_mode, - paths_for_files, - embedders_model, - hybrid_f0_method, -) - -def create_reference_tab(): - with gr.Row(): - gr.Markdown(translations["create_reference_markdown_2"]) - with gr.Row(): - pitch_guidance = gr.Checkbox( - label=translations["training_pitch"], - value=True, - interactive=True - ) - use_energy = gr.Checkbox( - label=translations["train&energy"], - value=False, - interactive=True - ) - f0_autotune = gr.Checkbox( - label=translations["autotune"], - value=False, - interactive=True - ) - proposal_pitch = gr.Checkbox( - label=translations["proposal_pitch"], - value=False, - interactive=True - ) - with gr.Row(): - create_reference_button = gr.Button( - translations["create_reference"], - variant="primary" - ) - with gr.Row(): - f0_up_key = gr.Slider( - minimum=-24, - maximum=24, - step=1, - info=translations["pitch_info"], - label=translations["pitch"], - value=0, - interactive=True - ) - proposal_pitch_threshold = gr.Slider( - minimum=50.0, - maximum=1200.0, - label=translations["proposal_pitch_threshold"], - info=translations["proposal_pitch_threshold_info"], - value=255.0, - step=0.1, - interactive=True, - visible=False - ) - with gr.Row(): - filter_radius = gr.Slider( - minimum=0, - maximum=7, - label=translations["filter_radius"], - info=translations["filter_radius_info"], - value=3, - step=1, - interactive=True - ) - f0_autotune_strength = gr.Slider( - minimum=0, - maximum=1, - label=translations["autotune_rate"], - info=translations["autotune_rate_info"], - value=1, - step=0.1, - interactive=True, - visible=False - ) - with gr.Row(): - with gr.Column(): - with gr.Accordion( - translations["input_output"], - open=False - ): - with gr.Column(): - input_audio = gr.Dropdown( - label=translations["audio_path"], - value="", - choices=paths_for_files, - info=translations["provide_audio"], - allow_custom_value=True, - interactive=True - ) - reference_name = gr.Textbox( - label=translations["reference_name"], - value="reference", - placeholder="reference", - info=translations["reference_name_info"], - interactive=True - ) - with gr.Column(): - refresh_audio = gr.Button( - translations["refresh"] - ) - with gr.Column(): - upload_audio = gr.Files( - label=translations["drop_audio"], - file_types=file_types - ) - with gr.Column(): - play_audio = gr.Audio( - interactive=False, - label=translations["input_audio"], - **audio_params - ) - with gr.Column() as f0_method_column: - with gr.Accordion( - label=translations["f0_method"], - open=False - ): - with gr.Group(): - with gr.Row(): - predictor_onnx = gr.Checkbox( - label=translations["predictor_onnx"], - value=False, - interactive=True - ) - unlock_full_method = gr.Checkbox( - label=translations["f0_unlock"], - value=False, - interactive=True - ) - f0_method = gr.Radio( - label=translations["f0_method"], - info=translations["f0_method_info"], - choices=method_f0, - value="rmvpe", - interactive=True - ) - f0_hybrid_method = gr.Dropdown( - label=translations["f0_method_hybrid"], - info=translations["f0_method_hybrid_info"], - choices=hybrid_f0_method, - value=hybrid_f0_method[0], - interactive=True, - allow_custom_value=True, - visible=False - ) - with gr.Row(): - alpha = gr.Slider( - label=translations["alpha_label"], - info=translations["alpha_info"], - minimum=0.1, - maximum=1, - value=0.5, - step=0.1, - interactive=True, - visible=False - ) - with gr.Column(): - with gr.Accordion( - label=translations["hubert_model"], - open=False - ): - embedders_mix = gr.Checkbox( - label=translations["embedders_mix"], - info=translations["embedders_mix_info"], - value=False, - interactive=True - ) - with gr.Row(): - version = gr.Radio( - label=translations["training_version"], - info=translations["training_version_info"], - choices=[ - "v1", - "v2" - ], - value="v2", - interactive=True - ) - with gr.Group(): - embedder_mode = gr.Radio( - label=translations["embed_mode"], - info=translations["embed_mode_info"], - value="fairseq", - choices=embedders_mode, - interactive=True, - visible=True - ) - embedders = gr.Radio( - label=translations["hubert_model"], - info=translations["hubert_info"], - choices=embedders_model, - value="hubert_base", - interactive=True - ) - with gr.Row(): - embedders_custom = gr.Textbox( - label=translations["modelname"], - info=translations["modelname_info"], - value="", - placeholder="hubert_base", - interactive=True, - visible=False - ) - with gr.Column(visible=False) as embedders_mix_column: - embedders_mix_layers = gr.Slider( - label=translations["embedders_mix_layers"], - info=translations["embedders_mix_layers_info"], - minimum=1, - maximum=12, - value=9, - step=1, - interactive=True - ) - embedders_mix_ratio = gr.Slider( - label=translations["embedders_mix_ratio"], - info=translations["embedders_mix_ratio_info"], - minimum=0.1, - maximum=1, - value=0.5, - step=0.1, - interactive=True - ) - with gr.Row(): - create_reference_info = gr.Textbox( - label=translations["reference_info"], - value="", - interactive=False, - lines=2 - ) - with gr.Row(): - f0_autotune.change( - fn=visible, - inputs=[ - f0_autotune - ], - outputs=[ - f0_autotune_strength - ] - ) - proposal_pitch.change( - fn=visible, - inputs=[ - proposal_pitch - ], - outputs=[ - proposal_pitch_threshold - ] - ) - unlock_full_method.change( - fn=unlock_f0, - inputs=[ - unlock_full_method - ], - outputs=[ - f0_method - ] - ) - with gr.Row(): - input_audio.change( - fn=lambda audio: audio, - inputs=[ - input_audio - ], - outputs=[ - play_audio - ] - ) - refresh_audio.click( - fn=change_audios_choices, - inputs=[ - input_audio - ], - outputs=[ - input_audio - ] - ) - f0_method.change( - fn=lambda method: [ - visible(method == "hybrid") - for _ in range(2) - ], - inputs=[ - f0_method - ], - outputs=[ - f0_hybrid_method, - alpha - ] - ) - with gr.Row(): - upload_audio.upload( - fn=lambda audio_in: [ - shutil_move(audio.name, configs["audios_path"]) - for audio in audio_in - ][0], - inputs=[ - upload_audio - ], - outputs=[ - input_audio - ] - ) - embedder_mode.change( - fn=change_embedders_mode, - inputs=[ - embedder_mode - ], - outputs=[ - embedders - ] - ) - embedders.change( - fn=lambda embedders: visible(embedders == "custom"), - inputs=[ - embedders - ], - outputs=[ - embedders_custom - ] - ) - with gr.Row(): - pitch_guidance.change( - fn=visible, - inputs=[ - pitch_guidance - ], - outputs=[ - f0_method_column - ] - ) - embedders_mix.change( - fn=visible, - inputs=[ - embedders_mix - ], - outputs=[ - embedders_mix_column - ] - ) - create_reference_button.click( - fn=create_reference, - inputs=[ - input_audio, - reference_name, - pitch_guidance, - use_energy, - version, - embedders, - embedder_mode, - f0_method, - predictor_onnx, - f0_up_key, - filter_radius, - f0_autotune, - f0_autotune_strength, - proposal_pitch, - proposal_pitch_threshold, - alpha, - embedders_mix, - embedders_mix_layers, - embedders_mix_ratio - ], - outputs=[ - create_reference_info - ], - api_name="create_reference" - ) \ No newline at end of file diff --git a/main/app/tabs/training/child/training.py b/main/app/tabs/training/child/training.py deleted file mode 100644 index defc2ae3d7d8596a9099c4adad8c2dd872b1eedd..0000000000000000000000000000000000000000 --- a/main/app/tabs/training/child/training.py +++ /dev/null @@ -1,1020 +0,0 @@ -import os -import sys - -import gradio as gr - -sys.path.append(os.getcwd()) - -from main.app.core.process import zip_file - -from main.app.core.training import ( - extract, - training, - preprocess, - create_index -) - -from main.app.variables import ( - config, - method_f0, - file_types, - model_name, - index_path, - pretrainedG, - pretrainedD, - translations, - reference_list, - embedders_mode, - embedders_model, - hybrid_f0_method -) - -from main.app.core.ui import ( - visible, - unlock_f0, - gr_warning, - shutil_move, - get_gpu_info, - vocoders_lock, - unlock_version, - unlock_vocoder, - hoplength_show, - gpu_number_str, - pitch_guidance_lock, - change_models_choices, - change_embedders_mode, - change_reference_choices, - change_pretrained_choices -) - -def training_model_tab(): - with gr.Row(): - gr.Markdown(translations["training_markdown"]) - with gr.Row(): - with gr.Column(): - with gr.Row(): - with gr.Column(): - modelname = gr.Textbox( - label=translations["modelname"], - info=translations["training_model_name"], - value="", - placeholder=translations["modelname"], - interactive=True - ) - sample_rate = gr.Radio( - label=translations["sample_rate"], - info=translations["sample_rate_info"], - choices=[ - "24k", - "32k", - "40k", - "44.1k", - "48k" - ], - value="48k", - interactive=True - ) - rvc_version = gr.Radio( - label=translations["training_version"], - info=translations["training_version_info"], - choices=[ - "v1", - "v2" - ], - value="v2", - interactive=True - ) - with gr.Group(): - with gr.Row(): - clean_dataset = gr.Checkbox( - label=translations["clear_dataset"], - value=False, - interactive=True - ) - process_effects = gr.Checkbox( - label=translations["preprocess_effect"], - value=False, - interactive=True - ) - pitch_guidance = gr.Checkbox( - label=translations["training_pitch"], - value=True, - interactive=True - ) - with gr.Row(): - custom_reference = gr.Checkbox( - label=translations["custom_reference"], - value=False, - interactive=True - ) - checkpointing = gr.Checkbox( - label=translations["memory_efficient_training"], - value=False, - interactive=True - ) - dataset_upload = gr.Checkbox( - label=translations["upload_dataset"], - value=False, - interactive=True - ) - with gr.Row(): - split_audio_mode = gr.Radio( - label=translations["split_audio_mode"], - info=translations["split_audio_mode_info"], - value="Automatic", - choices=[ - "Automatic", - "Simple", - "Skip" - ], - interactive=True - ) - normalization_mode = gr.Radio( - label=translations["normalization_mode"], - info=translations["normalization_mode_info"], - value="post", - choices=[ - "none", - "pre", - "post" - ], - interactive=True - ) - with gr.Row(visible=False) as custom_reference_row: - with gr.Accordion( - translations["custom_reference"], - open=True - ): - reference_name = gr.Dropdown( - label=translations["reference_name"], - info=translations["reference_name_info"], - choices=reference_list, - value=reference_list[0] if len(reference_list) >= 1 else "", - allow_custom_value=True, - interactive=True - ) - reference_refresh = gr.Button( - translations["refresh"], - scale=2 - ) - with gr.Row(visible=False) as clean_dataset_row: - clean_dataset_strength = gr.Slider( - label=translations["clean_strength"], - info=translations["clean_strength_info"], - minimum=0, - maximum=1, - value=0.7, - step=0.1, - interactive=True - ) - with gr.Column(): - preprocess_button = gr.Button( - translations["preprocess_button"], - scale=2 - ) - upload_dataset = gr.Files( - label=translations["drop_audio"], - file_types=file_types, - visible=False - ) - preprocess_info = gr.Textbox( - label=translations["preprocess_info"], - value="", - interactive=False, - container=True, - lines=2 - ) - with gr.Column(): - with gr.Row(): - with gr.Column(): - with gr.Accordion( - label=translations["f0_method"], - open=False - ): - with gr.Group(): - with gr.Row(): - predictor_onnx = gr.Checkbox( - label=translations["predictor_onnx"], - value=False, - interactive=True - ) - unlock_full_method = gr.Checkbox( - label=translations["f0_unlock"], - value=False, - interactive=True - ) - autotune = gr.Checkbox( - label=translations["autotune"], - value=False, - interactive=True - ) - f0_method = gr.Radio( - label=translations["f0_method"], - info=translations["f0_method_info"], - choices=method_f0, - value="rmvpe", - interactive=True - ) - hybrid_f0method = gr.Dropdown( - label=translations["f0_method_hybrid"], - info=translations["f0_method_hybrid_info"], - choices=hybrid_f0_method, - value=hybrid_f0_method[0], - interactive=True, - allow_custom_value=True, - visible=False - ) - hop_length = gr.Slider( - label=translations['hop_length'], - info=translations["hop_length_info"], - minimum=64, - maximum=512, - value=160, - step=1, - interactive=True, - visible=False - ) - f0_autotune_strength = gr.Slider( - label=translations["autotune_rate"], - info=translations["autotune_rate_info"], - minimum=0, - maximum=1, - value=1, - step=0.1, - interactive=True, - visible=False - ) - alpha = gr.Slider( - label=translations["alpha_label"], - info=translations["alpha_info"], - minimum=0.1, - maximum=1, - value=0.5, - step=0.1, - interactive=True, - visible=False - ) - with gr.Accordion( - label=translations["hubert_model"], - open=False - ): - embedders_mix = gr.Checkbox( - label=translations["embedders_mix"], - info=translations["embedders_mix_info"], - value=False, - interactive=True - ) - with gr.Group(): - embedder_mode = gr.Radio( - label=translations["embed_mode"], - info=translations["embed_mode_info"], - value="fairseq", - choices=embedders_mode, - interactive=True, - visible=True - ) - embedders = gr.Radio( - label=translations["hubert_model"], - info=translations["hubert_info"], - choices=embedders_model, - value="hubert_base", - interactive=True - ) - with gr.Row(): - embedders_custom = gr.Textbox( - label=translations["modelname"], - info=translations["modelname_info"], - value="", - placeholder="hubert_base", - interactive=True, - visible=False - ) - with gr.Column(visible=False) as embedders_mix_column: - embedders_mix_layers = gr.Slider( - label=translations["embedders_mix_layers"], - info=translations["embedders_mix_layers_info"], - minimum=1, - maximum=12, - value=9, - step=1, - interactive=True - ) - embedders_mix_ratio = gr.Slider( - label=translations["embedders_mix_ratio"], - info=translations["embedders_mix_ratio_info"], - minimum=0.1, - maximum=1, - value=0.5, - step=0.1, - interactive=True - ) - with gr.Column(): - extract_button = gr.Button( - translations["extract_button"], - scale=2 - ) - extract_info = gr.Textbox( - label=translations["extract_info"], - value="", - interactive=False, - lines=2 - ) - with gr.Column(): - with gr.Row(): - with gr.Column(): - total_epochs = gr.Slider( - label=translations["total_epoch"], - info=translations["total_epoch_info"], - minimum=1, - maximum=10000, - value=300, - step=1, - interactive=True - ) - save_epochs = gr.Slider( - label=translations["save_epoch"], - info=translations["save_epoch_info"], - minimum=1, - maximum=10000, - value=50, - step=1, - interactive=True - ) - with gr.Column(): - create_index_button = gr.Button( - f"3. {translations['create_index']}", - variant="primary", - scale=2 - ) - training_button = gr.Button( - f"4. {translations['training_model']}", - variant="primary", - scale=2 - ) - with gr.Row(): - with gr.Accordion( - label=translations["setting"], - open=False - ): - with gr.Row(): - index_algorithm = gr.Radio( - label=translations["index_algorithm"], - info=translations["index_algorithm_info"], - choices=[ - "Auto", - "Faiss", - "KMeans" - ], - value="Auto", - interactive=True - ) - with gr.Row(): - nprobe = gr.Slider( - label=translations["nprobe"], - info=translations["nprobe_info"], - minimum=1, - maximum=64, - value=9, - step=1, - interactive=True - ) - with gr.Row(): - cache_in_gpu = gr.Checkbox( - label=translations["cache_in_gpu"], - info=translations["cache_in_gpu_info"], - value=True, - interactive=True - ) - energy = gr.Checkbox( - label=translations["train&energy"], - info=translations["train&energy_info"], - value=False, - interactive=True - ) - overtraining_detector = gr.Checkbox( - label=translations["overtraining_detector"], - info=translations["overtraining_detector_info"], - value=False, - interactive=True - ) - with gr.Row(): - custom_dataset = gr.Checkbox( - label=translations["custom_dataset"], - info=translations["custom_dataset_info"], - value=False, - interactive=True - ) - save_only_latest = gr.Checkbox( - label=translations["save_only_latest"], - info=translations["save_only_latest_info"], - value=True, - interactive=True - ) - save_every_weights = gr.Checkbox( - label=translations["save_every_weights"], - info=translations["save_every_weights_info"], - value=True, - interactive=True - ) - with gr.Row(): - cleanup_training = gr.Checkbox( - label=translations["cleanup_training"], - info=translations["cleanup_training_info"], - value=False, - interactive=True - ) - not_use_pretrain = gr.Checkbox( - label=translations["not_use_pretrain_2"], - info=translations["not_use_pretrain_info"], - value=False, - interactive=True - ) - custom_pretrain = gr.Checkbox( - label=translations["custom_pretrain"], - info=translations["custom_pretrain_info"], - value=False, - interactive=True - ) - with gr.Column(): - dataset_path = gr.Textbox( - label=translations["dataset_folder"], - value="dataset", - interactive=True, - visible=False - ) - with gr.Column(): - include_mutes = gr.Slider( - minimum=0, - maximum=10, - value=2, - step=1, - label=translations["include_mutes"], - info=translations["include_mutes_info"], - interactive=True - ) - with gr.Row(visible=False) as simple_option: - chunk_len = gr.Slider( - minimum=0.5, - maximum=5.0, - value=3.0, - step=0.1, - label=translations["chunk_length"], - info=translations["chunk_length_info"], - interactive=True - ) - overlap_len = gr.Slider( - minimum=0.0, - maximum=0.4, - value=0.3, - step=0.1, - label=translations["overlap_length"], - info=translations["overlap_length_info"], - interactive=True - ) - threshold = gr.Slider( - minimum=1, - maximum=100, - value=50, - step=1, - label=translations["threshold"], - info=translations["overtraining_threshold"], - interactive=True, - visible=False - ) - with gr.Accordion( - translations["setting_cpu_gpu"], - open=False - ): - with gr.Row(): - architecture = gr.Radio( - label=translations["architecture"], - info=translations["architecture_info"], - choices=[ - "RVC", - "SVC" - ], - value="RVC", - interactive=True - ) - with gr.Column(): - gpu_number = gr.Textbox( - label=translations["gpu_number"], - value=gpu_number_str(), - info=translations["gpu_number_info"], - interactive=True - ) - gpu_str, gpu_len = get_gpu_info() - gr.Textbox( - label=translations["gpu_info"], - value=gpu_str, - info=translations["gpu_info_2"], - interactive=False, - lines=gpu_len - ) - cpu_core = gr.Slider( - label=translations["cpu_core"], - info=translations["cpu_core_info"], - minimum=1, - maximum=os.cpu_count(), - value=min(os.cpu_count(), 4), - step=1, - interactive=True - ) - batch_size = gr.Slider( - label=translations["batch_size"], - info=translations["batch_size_info"], - minimum=1, - maximum=64, - value=8, - step=1, - interactive=True - ) - with gr.Group(): - multiscale_mel_loss = gr.Checkbox( - label=translations["multiscale_mel_loss"], - info=translations["multiscale_mel_loss_info"], - value=False, - interactive=True - ) - cosine_annealing_lr = gr.Checkbox( - label=translations["cosine_annealing_lr"], - info=translations["cosine_annealing_lr_info"], - value=False, - interactive=True - ) - vocoders = gr.Radio( - label=translations["vocoder"], - info=translations["vocoder_info"], - choices=[ - "Default", - "MRF-HiFi-GAN", - "RefineGAN", - "BigVGAN" - ], - value="Default", - interactive=True - ) - with gr.Row(): - deterministic = gr.Checkbox( - label=translations["deterministic"], - info=translations["deterministic_info"], - value=False, - interactive=config.device.startswith("cuda") and not config.is_zluda - ) - benchmark = gr.Checkbox( - label=translations["benchmark"], - info=translations["benchmark_info"], - value=False, - interactive=config.device.startswith("cuda") and not config.is_zluda - ) - with gr.Row(): - optimizer = gr.Radio( - label=translations["optimizer"], - info=translations["optimizer_info"], - value="AdamW", - choices=[ - "AdamW", - "RAdam", - "AnyPrecisionAdamW", - "AdaBelief", - "AdaBeliefV2" - ], - interactive=True - ) - with gr.Row(): - model_author = gr.Textbox( - label=translations["training_author"], - info=translations["training_author_info"], - value="", - placeholder=translations["training_author"], - interactive=True - ) - with gr.Row(): - with gr.Column(): - with gr.Accordion( - translations["custom_pretrain_info"], - open=False, - visible=False - ) as pretrain_setting: - pretrained_D = gr.Dropdown( - label=translations["pretrain_file"].format(dg="D"), - choices=pretrainedD, - value=pretrainedD[0] if len(pretrainedD) > 0 else '', - interactive=True, - allow_custom_value=True - ) - pretrained_G = gr.Dropdown( - label=translations["pretrain_file"].format(dg="G"), - choices=pretrainedG, - value=pretrainedG[0] if len(pretrainedG) > 0 else '', - interactive=True, - allow_custom_value=True - ) - pretrained_refresh = gr.Button( - translations["refresh"], - scale=2 - ) - with gr.Row(): - training_info = gr.Textbox( - label=translations["train_info"], - value="", - interactive=False, - lines=3 - ) - with gr.Row(): - with gr.Column(): - with gr.Accordion( - translations["export_model"], - open=False - ): - with gr.Row(): - model_file = gr.Dropdown( - label=translations["model_name"], - choices=model_name, - value=model_name[0] if len(model_name) >= 1 else "", - interactive=True, - allow_custom_value=True - ) - index_file = gr.Dropdown( - label=translations["index_path"], - choices=index_path, - value=index_path[0] if len(index_path) >= 1 else "", - interactive=True, - allow_custom_value=True - ) - with gr.Row(): - refresh_file = gr.Button( - f"1. {translations['refresh']}", - scale=2 - ) - zip_model = gr.Button( - translations["zip_model"], - variant="primary", - scale=2 - ) - with gr.Row(): - zip_output = gr.File( - label=translations["output_zip"], - file_types=[".zip"], - interactive=False, - visible=False - ) - with gr.Row(): - vocoders.change( - fn=pitch_guidance_lock, - inputs=[ - vocoders - ], - outputs=[ - pitch_guidance - ] - ) - pitch_guidance.change( - fn=vocoders_lock, - inputs=[ - pitch_guidance - ], - outputs=[ - vocoders - ] - ) - unlock_full_method.change( - fn=unlock_f0, - inputs=[ - unlock_full_method - ], - outputs=[ - f0_method - ] - ) - with gr.Row(): - refresh_file.click( - fn=change_models_choices, - inputs=[], - outputs=[ - model_file, - index_file - ] - ) - zip_model.click( - fn=zip_file, - inputs=[ - modelname, - model_file, - index_file - ], - outputs=[ - zip_output - ] - ) - dataset_path.change( - fn=lambda folder: os.makedirs(folder, exist_ok=True) if folder else None, - inputs=[ - dataset_path - ], - outputs=[] - ) - with gr.Row(): - dataset_upload.change( - fn=visible, - inputs=[ - dataset_upload - ], - outputs=[ - upload_dataset - ] - ) - overtraining_detector.change( - fn=visible, - inputs=[ - overtraining_detector - ], - outputs=[ - threshold - ] - ) - clean_dataset.change( - fn=visible, - inputs=[ - clean_dataset - ], - outputs=[ - clean_dataset_row - ] - ) - with gr.Row(): - custom_dataset.change( - fn=lambda custom_dataset: [ - visible(custom_dataset), - "dataset" - ], - inputs=[ - custom_dataset - ], - outputs=[ - dataset_path, - dataset_path - ] - ) - rvc_version.change( - fn=unlock_vocoder, - inputs=[ - rvc_version, - vocoders - ], - outputs=[ - vocoders - ] - ) - vocoders.change( - inputs=[ - rvc_version, - vocoders - ], - fn=unlock_version, - outputs=[ - rvc_version - ] - ) - with gr.Row(): - custom_reference.change( - fn=visible, - inputs=[ - custom_reference - ], - outputs=[ - custom_reference_row - ] - ) - f0_method.change( - fn=lambda method, hybrid: [ - visible(method == "hybrid"), - visible(method == "hybrid"), - hoplength_show(method, hybrid) - ], - inputs=[ - f0_method, - hybrid_f0method - ], - outputs=[ - hybrid_f0method, - alpha, - hop_length - ] - ) - hybrid_f0method.change( - fn=hoplength_show, - inputs=[ - f0_method, - hybrid_f0method - ], - outputs=[ - hop_length - ] - ) - with gr.Row(): - autotune.change( - fn=visible, - inputs=[ - autotune - ], - outputs=[ - f0_autotune_strength - ] - ) - split_audio_mode.change( - fn=lambda a: visible(a == "Simple"), - inputs=[ - split_audio_mode - ], - outputs=[ - simple_option - ] - ) - upload_dataset.upload( - fn=lambda files, folder: [ - shutil_move(f.name, os.path.join(folder, os.path.split(f.name)[1])) - for f in files - ] if folder != "" else gr_warning(translations["dataset_folder1"]), - inputs=[ - upload_dataset, - dataset_path - ], - outputs=[], - api_name="upload_dataset" - ) - with gr.Row(): - not_use_pretrain.change( - fn=lambda a, b: visible(a and not b), - inputs=[ - custom_pretrain, - not_use_pretrain - ], - outputs=[ - pretrain_setting - ] - ) - custom_pretrain.change( - fn=lambda a, b: visible(a and not b), - inputs=[ - custom_pretrain, - not_use_pretrain - ], - outputs=[ - pretrain_setting - ] - ) - pretrained_refresh.click( - fn=change_pretrained_choices, - inputs=[], - outputs=[ - pretrained_D, - pretrained_G - ] - ) - with gr.Row(): - embedder_mode.change( - fn=change_embedders_mode, - inputs=[ - embedder_mode - ], - outputs=[ - embedders - ] - ) - embedders.change( - fn=lambda embedders: visible(embedders == "custom"), - inputs=[ - embedders - ], - outputs=[ - embedders_custom - ] - ) - reference_refresh.click( - fn=change_reference_choices, - inputs=[], - outputs=[ - reference_name - ] - ) - with gr.Row(): - embedders_mix.change( - fn=visible, - inputs=[ - embedders_mix - ], - outputs=[ - embedders_mix_column - ] - ) - with gr.Row(): - preprocess_button.click( - fn=preprocess, - inputs=[ - modelname, - sample_rate, - cpu_core, - split_audio_mode, - process_effects, - dataset_path, - clean_dataset, - clean_dataset_strength, - chunk_len, - overlap_len, - normalization_mode, - architecture - ], - outputs=[ - preprocess_info - ], - api_name="preprocess" - ) - with gr.Row(): - extract_button.click( - fn=extract, - inputs=[ - modelname, - rvc_version, - f0_method, - pitch_guidance, - hop_length, - cpu_core, - gpu_number, - sample_rate, - embedders, - embedders_custom, - predictor_onnx, - embedder_mode, - autotune, - f0_autotune_strength, - hybrid_f0method, - energy, - alpha, - include_mutes, - embedders_mix, - embedders_mix_layers, - embedders_mix_ratio, - architecture - ], - outputs=[ - extract_info - ], - api_name="extract" - ) - with gr.Row(): - create_index_button.click( - fn=create_index, - inputs=[ - modelname, - rvc_version, - index_algorithm, - nprobe - ], - outputs=[ - training_info - ], - api_name="create_index" - ) - with gr.Row(): - training_button.click( - fn=training, - inputs=[ - modelname, - rvc_version, - save_epochs, - save_only_latest, - save_every_weights, - total_epochs, - sample_rate, - batch_size, - gpu_number, - pitch_guidance, - not_use_pretrain, - custom_pretrain, - pretrained_G, - pretrained_D, - overtraining_detector, - threshold, - cleanup_training, - cache_in_gpu, - model_author, - vocoders, - checkpointing, - deterministic, - benchmark, - optimizer, - energy, - custom_reference, - reference_name, - multiscale_mel_loss, - embedders, - embedders_custom, - cosine_annealing_lr, - architecture - ], - outputs=[ - training_info - ], - api_name="training_model" - ) \ No newline at end of file diff --git a/main/app/tabs/training/training.py b/main/app/tabs/training/training.py deleted file mode 100644 index 6f7714bb874e8d41d0df86a3aef3a8c5835ee676..0000000000000000000000000000000000000000 --- a/main/app/tabs/training/training.py +++ /dev/null @@ -1,34 +0,0 @@ -import os -import sys - -import gradio as gr - -sys.path.append(os.getcwd()) - -from main.app.variables import ( - configs, - translations -) - -def training_tab(): - with gr.TabItem(translations["training_model"]): - if configs.get("training_tab", True): - from main.app.tabs.training.child.training import training_model_tab - - with gr.TabItem(translations["training_model"]): - gr.Markdown(f"## {translations['training_model']}") - training_model_tab() - - if configs.get("create_dataset_tab", True): - from main.app.tabs.training.child.create_dataset import create_dataset_tab - - with gr.TabItem(translations["createdataset"]): - gr.Markdown(translations["create_dataset_markdown"]) - create_dataset_tab() - - if configs.get("create_reference_tab", True): - from main.app.tabs.training.child.create_reference import create_reference_tab - - with gr.TabItem(translations["create_reference"]): - gr.Markdown(translations["create_reference_markdown"]) - create_reference_tab() \ No newline at end of file diff --git a/main/app/variables.py b/main/app/variables.py deleted file mode 100644 index 07b9d3a222805da7c194fdd4c6e60f2cda860db1..0000000000000000000000000000000000000000 --- a/main/app/variables.py +++ /dev/null @@ -1,345 +0,0 @@ -import os -import re -import sys -import csv -import json -import spaces -import logging -import logging.handlers - -@spaces.GPU -def dummpy(): - pass - -import gradio as gr - -from packaging import version - -sys.path.append(os.getcwd()) - -from main.configs.config import Config - -logger = logging.getLogger(__name__) -logger.propagate = False - -config = Config() -python = sys.executable -translations = config.translations -configs_json = os.path.join("main", "configs", "config.json") -configs = json.load(open(configs_json, "r")) - -configs["debug_mode"] = "--debug" in sys.argv -configs["tensorrt"] = "--tensorrt" in sys.argv - -with open(configs_json, "w") as f: - json.dump(configs, f, indent=4) - -gradio_version = version.parse(gr.__version__) <= version.parse("6.0.0") - -audio_params = { - "show_download_button": True, - "show_share_button": True -} if gradio_version else { - "buttons": ["download", "share"] -} - -if not logger.hasHandlers(): - console_handler = logging.StreamHandler() - console_formatter = logging.Formatter(fmt="\n%(asctime)s.%(msecs)03d | %(levelname)s | %(module)s | %(message)s", datefmt="%Y-%m-%d %H:%M:%S") - console_handler.setFormatter(console_formatter) - console_handler.setLevel(logging.DEBUG if config.debug_mode else logging.INFO) - file_handler = logging.handlers.RotatingFileHandler(os.path.join(configs["logs_path"], "app.log"), maxBytes=10*1024*1024, backupCount=3, encoding='utf-8') - file_formatter = logging.Formatter(fmt="\n%(asctime)s.%(msecs)03d | %(levelname)s | %(module)s | %(message)s", datefmt="%Y-%m-%d %H:%M:%S") - file_handler.setFormatter(file_formatter) - file_handler.setLevel(logging.DEBUG) - logger.addHandler(console_handler) - logger.addHandler(file_handler) - logger.setLevel(logging.DEBUG) - -if config.device in ["cpu", "mps", "ocl:0", "privateuseone:0"] and configs.get("fp16", False) and not configs.get("allow_fp16_all_backend", False): - logger.warning(translations["fp16_not_support"]) - configs["fp16"] = config.is_half = False - - with open(configs_json, "w") as f: - json.dump(configs, f, indent=4) - -if configs.get("asio_enabled", False): - logger.debug(translations["enabled_asio"]) - os.environ["SD_ENABLE_ASIO"] = "1" - -models = {} -model_options = {} - -file_types = [ - ".wav", - ".mp3", - ".flac", - ".ogg", - ".opus", - ".m4a", - ".mp4", - ".aac", - ".alac", - ".wma", - ".aiff", - ".webm", - ".ac3" -] - -export_format_choices = [ - "wav", - "mp3", - "flac", - "ogg", - "opus", - "m4a", - "mp4", - "aac", - "alac", - "wma", - "aiff", - "webm", - "ac3" -] - -method_f0 = [ - "mangio-crepe-full", - "crepe-full", - "fcpe", - "rmvpe", - "harvest-stonemask", - "pyin", - "hybrid" -] - -method_f0_full = [ - "rmvpe-mix", - "rmvpe-mix-medfilt", - "hpa-rmvpe", - "hpa-rmvpe-medfilt", - "hpa-rmvpe-previous", - "hpa-rmvpe-previous-medfilt", - "rmvpe", - "rmvpe-medfilt", - "djcm", - "djcm-medfilt", - "djcm-svs", - "djcm-svs-medfilt", - "swift", - "fcpe", - "fcpe-previous", - "fcpe-legacy", - "mangio-crepe-full", - "crepe-full", - "mangio-crepe-large", - "crepe-large", - "mangio-crepe-medium", - "crepe-medium", - "mangio-crepe-small", - "crepe-small", - "mangio-crepe-tiny", - "crepe-tiny", - "harvest-stonemask", - "harvest", - "pesto", - "swipe-stonemask", - "swipe", - "mangio-penn", - "penn", - "dio-stonemask", - "dio", - "pm-ac", - "pm-cc", - "pm-shs", - "pyin", - "yin", - "piptrack", - "hybrid" -] - -hybrid_f0_method = [ - "hybrid[pm+dio]", - "hybrid[pm+crepe-tiny]", - "hybrid[pm+crepe]", - "hybrid[pm+fcpe]", - "hybrid[pm+rmvpe]", - "hybrid[pm+harvest]", - "hybrid[pm+yin]", - "hybrid[dio+crepe-tiny]", - "hybrid[dio+crepe]", - "hybrid[dio+fcpe]", - "hybrid[dio+rmvpe]", - "hybrid[dio+harvest]", - "hybrid[dio+yin]", - "hybrid[crepe-tiny+crepe]", - "hybrid[crepe-tiny+fcpe]", - "hybrid[crepe-tiny+rmvpe]", - "hybrid[crepe-tiny+harvest]", - "hybrid[crepe+fcpe]", - "hybrid[crepe+rmvpe]", - "hybrid[crepe+harvest]", - "hybrid[crepe+yin]", - "hybrid[fcpe+rmvpe]", - "hybrid[fcpe+harvest]", - "hybrid[fcpe+yin]", - "hybrid[rmvpe+harvest]", - "hybrid[rmvpe+yin]", - "hybrid[harvest+yin]" -] - -embedders_mode = [ - "fairseq", - "onnx", - "transformers", - "whisper" -] - -embedders_model = [ - "hubert_base", - "contentvec_base", - "vietnamese_hubert_base", - "japanese_hubert_base", - "korean_hubert_base", - "chinese_hubert_base", - "portuguese_hubert_base", - "spin-v1", - "spin-v2", - "custom" -] - -whisper_model = [ - "tiny", - "tiny.en", - "base", - "base.en", - "small", - "small.en", - "medium", - "medium.en", - "large-v1", - "large-v2", - "large-v3", - "large-v3-turbo" -] - -whisper_languages = configs.get("whisper_languages", ["vi", "en"]) - -sample_rate_choice = [ - 8000, - 11025, - 12000, - 16000, - 22050, - 24000, - 32000, - 44100, - 48000, - 88200, - 96000, - 176400, - 192000, - 352800, - 384000 -] - -paths_for_files = sorted([ - os.path.abspath(os.path.join(root, f)) - for root, _, files in os.walk(configs["audios_path"]) - for f in files - if os.path.splitext(f)[1].lower() in file_types and not f.startswith(("Convert", "output")) -]) - -reference_list = sorted([ - re.sub(r'_v\d+_(?:[A-Za-z0-9_]+?)_(True|False)_(True|False)$', '', name) - for name in os.listdir(configs["reference_path"]) - if ( - os.path.exists(os.path.join(configs["reference_path"], name)) and - os.path.isdir(os.path.join(configs["reference_path"], name)) - ) -]) - -model_name = sorted([ - model for model in os.listdir(configs["weights_path"]) - if model.endswith((".pth", ".onnx")) and not model.startswith("G_") and not model.startswith("D_") -]) - -index_path = sorted([ - os.path.join(root, name) - for root, _, files in os.walk(configs["logs_path"], topdown=False) - for name in files - if name.endswith(".index") and "trained" not in name -]) - -pretraineds = os.listdir(configs["pretrained_custom_path"]) - -pretrainedD = [ - model for model in pretraineds - if model.endswith(".pth") and "D" in model -] - -pretrainedG = [ - model for model in pretraineds - if model.endswith(".pth") and "G" in model -] - -all_presets = os.listdir(configs["presets_path"]) - -presets_file = sorted([ - f for f in all_presets - if f.endswith(".conversion.json") -]) - -audio_effect_presets_file = sorted([ - f for f in all_presets - if f.endswith(".effect.json") -]) - -realtime_presets_file = sorted([ - f for f in all_presets - if f.endswith(".realtime.json") -]) - -f0_file = sorted([ - os.path.abspath(os.path.join(root, f)) - for root, _, files in os.walk(configs["f0_path"]) - for f in files if f.endswith(".txt") -]) - -language = configs.get("language", "vi-VN") -theme = configs.get("theme", "NoCrypt/miku") - -edgetts = configs.get("edge_tts", ["vi-VN-HoaiMyNeural", "vi-VN-NamMinhNeural"]) -google_tts_voice = configs.get("google_tts_voice", ["vi", "en"]) - -vr_models = configs.get("vr_models", "") -demucs_models = configs.get("demucs_models", "") -mdx_models = configs.get("mdx_models", "") -karaoke_models = configs.get("karaoke_models", "") -reverb_models = configs.get("reverb_models", "") -denoise_models = configs.get("denoise_models", "") -uvr_model = list(demucs_models.keys()) + list(vr_models.keys()) + list(mdx_models.keys()) - -font = configs.get("font", "https://fonts.googleapis.com/css2?family=Saira&display=swap") -csv_path = configs["csv_path"] - -try: - if "--allow_all_disk" in sys.argv and sys.platform == "win32": - import win32api - allow_disk = win32api.GetLogicalDriveStrings().split('\x00')[:-1] - else: allow_disk = [] -except: - allow_disk = [] - -try: - for row in list(csv.DictReader(open(csv_path, newline='', encoding='utf-8'))): - filename = row['Filename'] - url = None - - for value in row.values(): - if isinstance(value, str) and "huggingface" in value: - url = value - break - - if url: models[filename] = url -except: - pass \ No newline at end of file diff --git a/main/configs/config.json b/main/configs/config.json deleted file mode 100644 index a8136b026f5b2834c6ba164e90fa3a3b904d2361..0000000000000000000000000000000000000000 --- a/main/configs/config.json +++ /dev/null @@ -1,741 +0,0 @@ -{ - "language": "vi-VN", - "support_language": [ - "en-US", - "vi-VN" - ], - "theme": "NoCrypt/miku", - "themes": [ - "NoCrypt/miku", - "gstaff/xkcd", - "JohnSmith9982/small_and_pretty", - "ParityError/Interstellar", - "earneleh/paris", - "shivi/calm_seafoam", - "Hev832/Applio", - "YTheme/Minecraft", - "gstaff/sketch", - "SebastianBravo/simci_css", - "allenai/gradio-theme", - "Nymbo/Nymbo_Theme_5", - "lone17/kotaemon", - "Zarkel/IBM_Carbon_Theme", - "SherlockRamos/Feliz", - "freddyaboulton/dracula_revamped", - "freddyaboulton/bad-theme-space", - "gradio/dracula_revamped", - "abidlabs/dracula_revamped", - "gradio/dracula_test", - "gradio/seafoam", - "gradio/glass", - "gradio/monochrome", - "gradio/soft", - "gradio/default", - "gradio/base", - "abidlabs/pakistan", - "dawood/microsoft_windows", - "ysharma/steampunk", - "ysharma/huggingface", - "abidlabs/Lime", - "freddyaboulton/this-theme-does-not-exist-2", - "aliabid94/new-theme", - "aliabid94/test2", - "aliabid94/test3", - "aliabid94/test4", - "abidlabs/banana", - "freddyaboulton/test-blue", - "gstaff/whiteboard", - "ysharma/llamas", - "abidlabs/font-test", - "YenLai/Superhuman", - "bethecloud/storj_theme", - "sudeepshouche/minimalist", - "knotdgaf/gradiotest", - "ParityError/Anime", - "Ajaxon6255/Emerald_Isle", - "ParityError/LimeFace", - "finlaymacklon/smooth_slate", - "finlaymacklon/boxy_violet", - "derekzen/stardust", - "EveryPizza/Cartoony-Gradio-Theme", - "Ifeanyi/Cyanister", - "Tshackelton/IBMPlex-DenseReadable", - "snehilsanyal/scikit-learn", - "Himhimhim/xkcd", - "nota-ai/theme", - "rawrsor1/Everforest", - "rottenlittlecreature/Moon_Goblin", - "abidlabs/test-yellow", - "abidlabs/test-yellow3", - "idspicQstitho/dracula_revamped", - "kfahn/AnimalPose", - "HaleyCH/HaleyCH_Theme", - "simulKitke/dracula_test", - "braintacles/CrimsonNight", - "wentaohe/whiteboardv2", - "reilnuud/polite", - "remilia/Ghostly", - "Franklisi/darkmode", - "coding-alt/soft", - "xiaobaiyuan/theme_land", - "step-3-profit/Midnight-Deep", - "xiaobaiyuan/theme_demo", - "Taithrah/Minimal", - "Insuz/SimpleIndigo", - "zkunn/Alipay_Gradio_theme", - "Insuz/Mocha", - "xiaobaiyuan/theme_brief", - "Ama434/434-base-Barlow", - "Ama434/def_barlow", - "Ama434/neutral-barlow", - "dawood/dracula_test", - "nuttea/Softblue", - "BlueDancer/Alien_Diffusion", - "naughtondale/monochrome", - "Dagfinn1962/standard", - "default" - ], - "mdx_models": { - "Main_340": "UVR-MDX-NET_Main_340.onnx", - "Main_390": "UVR-MDX-NET_Main_390.onnx", - "Main_406": "UVR-MDX-NET_Main_406.onnx", - "Main_427": "UVR-MDX-NET_Main_427.onnx", - "Main_438": "UVR-MDX-NET_Main_438.onnx", - "Inst_full_292": "UVR-MDX-NET-Inst_full_292.onnx", - "Inst_HQ_1": "UVR-MDX-NET-Inst_HQ_1.onnx", - "Inst_HQ_2": "UVR-MDX-NET-Inst_HQ_2.onnx", - "Inst_HQ_3": "UVR-MDX-NET-Inst_HQ_3.onnx", - "Inst_HQ_4": "UVR-MDX-NET-Inst_HQ_4.onnx", - "Inst_HQ_5": "UVR-MDX-NET-Inst_HQ_5.onnx", - "Kim_Vocal_1": "Kim_Vocal_1.onnx", - "Kim_Vocal_2": "Kim_Vocal_2.onnx", - "Kim_Inst": "Kim_Inst.onnx", - "Inst_187_beta": "UVR-MDX-NET_Inst_187_beta.onnx", - "Inst_82_beta": "UVR-MDX-NET_Inst_82_beta.onnx", - "Inst_90_beta": "UVR-MDX-NET_Inst_90_beta.onnx", - "Voc_FT": "UVR-MDX-NET-Voc_FT.onnx", - "Crowd_HQ": "UVR-MDX-NET_Crowd_HQ_1.onnx", - "MDXNET_9482": "UVR_MDXNET_9482.onnx", - "Inst_1": "UVR-MDX-NET-Inst_1.onnx", - "Inst_2": "UVR-MDX-NET-Inst_2.onnx", - "Inst_3": "UVR-MDX-NET-Inst_3.onnx", - "MDXNET_1_9703": "UVR_MDXNET_1_9703.onnx", - "MDXNET_2_9682": "UVR_MDXNET_2_9682.onnx", - "MDXNET_3_9662": "UVR_MDXNET_3_9662.onnx", - "Inst_Main": "UVR-MDX-NET-Inst_Main.onnx", - "MDXNET_Main": "UVR_MDXNET_Main.onnx" - }, - "demucs_models": { - "HT-Tuned": "htdemucs_ft.pth", - "HT-Normal": "htdemucs.pth", - "HD_MMI": "hdemucs_mmi.pth", - "HT_6S": "htdemucs_6s.pth" - }, - "vr_models": { - "HP-1": "1_HP-UVR.pth", - "HP-2": "2_HP-UVR.pth", - "HP-Vocal-1": "3_HP-Vocal-UVR.pth", - "HP-Vocal-2": "4_HP-Vocal-UVR.pth", - "HP2-1": "7_HP2-UVR.pth", - "HP2-2": "8_HP2-UVR.pth", - "HP2-3": "9_HP2-UVR.pth", - "SP-2B-1": "10_SP-UVR-2B-32000-1.pth", - "SP-2B-2": "11_SP-UVR-2B-32000-2.pth", - "SP-3B-1": "12_SP-UVR-3B-44100.pth", - "SP-4B-1": "13_SP-UVR-4B-44100-1.pth", - "SP-4B-2": "14_SP-UVR-4B-44100-2.pth", - "SP-MID-1": "15_SP-UVR-MID-44100-1.pth", - "SP-MID-2": "16_SP-UVR-MID-44100-2.pth" - }, - "karaoke_models": { - "MDX-Version-1": "UVR_MDXNET_KARA.onnx", - "MDX-Version-2": "UVR_MDXNET_KARA_2.onnx", - "VR-Version-1": "5_HP-Karaoke-UVR.pth", - "VR-Version-2": "6_HP-Karaoke-UVR.pth" - }, - "reverb_models": { - "MDX-Reverb": "Reverb_HQ_By_FoxJoy.onnx", - "VR-Reverb": "UVR-DeEcho-DeReverb.pth", - "Echo-Aggressive": "UVR-De-Echo-Aggressive.pth", - "Echo-Normal": "UVR-De-Echo-Normal.pth" - }, - "denoise_models": { - "Lite": "UVR-DeNoise-Lite.pth", - "Normal": "UVR-DeNoise.pth" - }, - "edge_tts": [ - "af-ZA-AdriNeural", - "af-ZA-WillemNeural", - "sq-AL-AnilaNeural", - "sq-AL-IlirNeural", - "am-ET-AmehaNeural", - "am-ET-MekdesNeural", - "ar-DZ-AminaNeural", - "ar-DZ-IsmaelNeural", - "ar-BH-AliNeural", - "ar-BH-LailaNeural", - "ar-EG-SalmaNeural", - "ar-EG-ShakirNeural", - "ar-IQ-BasselNeural", - "ar-IQ-RanaNeural", - "ar-JO-SanaNeural", - "ar-JO-TaimNeural", - "ar-KW-FahedNeural", - "ar-KW-NouraNeural", - "ar-LB-LaylaNeural", - "ar-LB-RamiNeural", - "ar-LY-ImanNeural", - "ar-LY-OmarNeural", - "ar-MA-JamalNeural", - "ar-MA-MounaNeural", - "ar-OM-AbdullahNeural", - "ar-OM-AyshaNeural", - "ar-QA-AmalNeural", - "ar-QA-MoazNeural", - "ar-SA-HamedNeural", - "ar-SA-ZariyahNeural", - "ar-SY-AmanyNeural", - "ar-SY-LaithNeural", - "ar-TN-HediNeural", - "ar-TN-ReemNeural", - "ar-AE-FatimaNeural", - "ar-AE-HamdanNeural", - "ar-YE-MaryamNeural", - "ar-YE-SalehNeural", - "az-AZ-BabekNeural", - "az-AZ-BanuNeural", - "bn-BD-NabanitaNeural", - "bn-BD-PradeepNeural", - "bn-IN-BashkarNeural", - "bn-IN-TanishaaNeural", - "bs-BA-GoranNeural", - "bs-BA-VesnaNeural", - "bg-BG-BorislavNeural", - "bg-BG-KalinaNeural", - "my-MM-NilarNeural", - "my-MM-ThihaNeural", - "ca-ES-EnricNeural", - "ca-ES-JoanaNeural", - "zh-HK-HiuGaaiNeural", - "zh-HK-HiuMaanNeural", - "zh-HK-WanLungNeural", - "zh-CN-XiaoxiaoNeural", - "zh-CN-XiaoyiNeural", - "zh-CN-YunjianNeural", - "zh-CN-YunxiNeural", - "zh-CN-YunxiaNeural", - "zh-CN-YunyangNeural", - "zh-CN-liaoning-XiaobeiNeural", - "zh-TW-HsiaoChenNeural", - "zh-TW-YunJheNeural", - "zh-TW-HsiaoYuNeural", - "zh-CN-shaanxi-XiaoniNeural", - "hr-HR-GabrijelaNeural", - "hr-HR-SreckoNeural", - "cs-CZ-AntoninNeural", - "cs-CZ-VlastaNeural", - "da-DK-ChristelNeural", - "da-DK-JeppeNeural", - "nl-BE-ArnaudNeural", - "nl-BE-DenaNeural", - "nl-NL-ColetteNeural", - "nl-NL-FennaNeural", - "nl-NL-MaartenNeural", - "en-AU-NatashaNeural", - "en-AU-WilliamNeural", - "en-CA-ClaraNeural", - "en-CA-LiamNeural", - "en-HK-SamNeural", - "en-HK-YanNeural", - "en-IN-NeerjaExpressiveNeural", - "en-IN-NeerjaNeural", - "en-IN-PrabhatNeural", - "en-IE-ConnorNeural", - "en-IE-EmilyNeural", - "en-KE-AsiliaNeural", - "en-KE-ChilembaNeural", - "en-NZ-MitchellNeural", - "en-NZ-MollyNeural", - "en-NG-AbeoNeural", - "en-NG-EzinneNeural", - "en-PH-JamesNeural", - "en-PH-RosaNeural", - "en-SG-LunaNeural", - "en-SG-WayneNeural", - "en-ZA-LeahNeural", - "en-ZA-LukeNeural", - "en-TZ-ElimuNeural", - "en-TZ-ImaniNeural", - "en-GB-LibbyNeural", - "en-GB-MaisieNeural", - "en-GB-RyanNeural", - "en-GB-SoniaNeural", - "en-GB-ThomasNeural", - "en-US-AvaMultilingualNeural", - "en-US-AndrewMultilingualNeural", - "en-US-EmmaMultilingualNeural", - "en-US-BrianMultilingualNeural", - "en-US-AvaNeural", - "en-US-AndrewNeural", - "en-US-EmmaNeural", - "en-US-BrianNeural", - "en-US-AnaNeural", - "en-US-AriaNeural", - "en-US-ChristopherNeural", - "en-US-EricNeural", - "en-US-GuyNeural", - "en-US-JennyNeural", - "en-US-MichelleNeural", - "en-US-RogerNeural", - "en-US-SteffanNeural", - "et-EE-AnuNeural", - "et-EE-KertNeural", - "fil-PH-AngeloNeural", - "fil-PH-BlessicaNeural", - "fi-FI-HarriNeural", - "fi-FI-NooraNeural", - "fr-BE-CharlineNeural", - "fr-BE-GerardNeural", - "fr-CA-ThierryNeural", - "fr-CA-AntoineNeural", - "fr-CA-JeanNeural", - "fr-CA-SylvieNeural", - "fr-FR-VivienneMultilingualNeural", - "fr-FR-RemyMultilingualNeural", - "fr-FR-DeniseNeural", - "fr-FR-EloiseNeural", - "fr-FR-HenriNeural", - "fr-CH-ArianeNeural", - "fr-CH-FabriceNeural", - "gl-ES-RoiNeural", - "gl-ES-SabelaNeural", - "ka-GE-EkaNeural", - "ka-GE-GiorgiNeural", - "de-AT-IngridNeural", - "de-AT-JonasNeural", - "de-DE-SeraphinaMultilingualNeural", - "de-DE-FlorianMultilingualNeural", - "de-DE-AmalaNeural", - "de-DE-ConradNeural", - "de-DE-KatjaNeural", - "de-DE-KillianNeural", - "de-CH-JanNeural", - "de-CH-LeniNeural", - "el-GR-AthinaNeural", - "el-GR-NestorasNeural", - "gu-IN-DhwaniNeural", - "gu-IN-NiranjanNeural", - "he-IL-AvriNeural", - "he-IL-HilaNeural", - "hi-IN-MadhurNeural", - "hi-IN-SwaraNeural", - "hu-HU-NoemiNeural", - "hu-HU-TamasNeural", - "is-IS-GudrunNeural", - "is-IS-GunnarNeural", - "id-ID-ArdiNeural", - "id-ID-GadisNeural", - "ga-IE-ColmNeural", - "ga-IE-OrlaNeural", - "it-IT-GiuseppeNeural", - "it-IT-DiegoNeural", - "it-IT-ElsaNeural", - "it-IT-IsabellaNeural", - "ja-JP-KeitaNeural", - "ja-JP-NanamiNeural", - "jv-ID-DimasNeural", - "jv-ID-SitiNeural", - "kn-IN-GaganNeural", - "kn-IN-SapnaNeural", - "kk-KZ-AigulNeural", - "kk-KZ-DauletNeural", - "km-KH-PisethNeural", - "km-KH-SreymomNeural", - "ko-KR-HyunsuNeural", - "ko-KR-InJoonNeural", - "ko-KR-SunHiNeural", - "lo-LA-ChanthavongNeural", - "lo-LA-KeomanyNeural", - "lv-LV-EveritaNeural", - "lv-LV-NilsNeural", - "lt-LT-LeonasNeural", - "lt-LT-OnaNeural", - "mk-MK-AleksandarNeural", - "mk-MK-MarijaNeural", - "ms-MY-OsmanNeural", - "ms-MY-YasminNeural", - "ml-IN-MidhunNeural", - "ml-IN-SobhanaNeural", - "mt-MT-GraceNeural", - "mt-MT-JosephNeural", - "mr-IN-AarohiNeural", - "mr-IN-ManoharNeural", - "mn-MN-BataaNeural", - "mn-MN-YesuiNeural", - "ne-NP-HemkalaNeural", - "ne-NP-SagarNeural", - "nb-NO-FinnNeural", - "nb-NO-PernilleNeural", - "ps-AF-GulNawazNeural", - "ps-AF-LatifaNeural", - "fa-IR-DilaraNeural", - "fa-IR-FaridNeural", - "pl-PL-MarekNeural", - "pl-PL-ZofiaNeural", - "pt-BR-ThalitaNeural", - "pt-BR-AntonioNeural", - "pt-BR-FranciscaNeural", - "pt-PT-DuarteNeural", - "pt-PT-RaquelNeural", - "ro-RO-AlinaNeural", - "ro-RO-EmilNeural", - "ru-RU-DmitryNeural", - "ru-RU-SvetlanaNeural", - "sr-RS-NicholasNeural", - "sr-RS-SophieNeural", - "si-LK-SameeraNeural", - "si-LK-ThiliniNeural", - "sk-SK-LukasNeural", - "sk-SK-ViktoriaNeural", - "sl-SI-PetraNeural", - "sl-SI-RokNeural", - "so-SO-MuuseNeural", - "so-SO-UbaxNeural", - "es-AR-ElenaNeural", - "es-AR-TomasNeural", - "es-BO-MarceloNeural", - "es-BO-SofiaNeural", - "es-CL-CatalinaNeural", - "es-CL-LorenzoNeural", - "es-ES-XimenaNeural", - "es-CO-GonzaloNeural", - "es-CO-SalomeNeural", - "es-CR-JuanNeural", - "es-CR-MariaNeural", - "es-CU-BelkysNeural", - "es-CU-ManuelNeural", - "es-DO-EmilioNeural", - "es-DO-RamonaNeural", - "es-EC-AndreaNeural", - "es-EC-LuisNeural", - "es-SV-LorenaNeural", - "es-SV-RodrigoNeural", - "es-GQ-JavierNeural", - "es-GQ-TeresaNeural", - "es-GT-AndresNeural", - "es-GT-MartaNeural", - "es-HN-CarlosNeural", - "es-HN-KarlaNeural", - "es-MX-DaliaNeural", - "es-MX-JorgeNeural", - "es-NI-FedericoNeural", - "es-NI-YolandaNeural", - "es-PA-MargaritaNeural", - "es-PA-RobertoNeural", - "es-PY-MarioNeural", - "es-PY-TaniaNeural", - "es-PE-AlexNeural", - "es-PE-CamilaNeural", - "es-PR-KarinaNeural", - "es-PR-VictorNeural", - "es-ES-AlvaroNeural", - "es-ES-ElviraNeural", - "es-US-AlonsoNeural", - "es-US-PalomaNeural", - "es-UY-MateoNeural", - "es-UY-ValentinaNeural", - "es-VE-PaolaNeural", - "es-VE-SebastianNeural", - "su-ID-JajangNeural", - "su-ID-TutiNeural", - "sw-KE-RafikiNeural", - "sw-KE-ZuriNeural", - "sw-TZ-DaudiNeural", - "sw-TZ-RehemaNeural", - "sv-SE-MattiasNeural", - "sv-SE-SofieNeural", - "ta-IN-PallaviNeural", - "ta-IN-ValluvarNeural", - "ta-MY-KaniNeural", - "ta-MY-SuryaNeural", - "ta-SG-AnbuNeural", - "ta-SG-VenbaNeural", - "ta-LK-KumarNeural", - "ta-LK-SaranyaNeural", - "te-IN-MohanNeural", - "te-IN-ShrutiNeural", - "th-TH-NiwatNeural", - "th-TH-PremwadeeNeural", - "tr-TR-AhmetNeural", - "tr-TR-EmelNeural", - "uk-UA-OstapNeural", - "uk-UA-PolinaNeural", - "ur-IN-GulNeural", - "ur-IN-SalmanNeural", - "ur-PK-AsadNeural", - "ur-PK-UzmaNeural", - "uz-UZ-MadinaNeural", - "uz-UZ-SardorNeural", - "vi-VN-HoaiMyNeural", - "vi-VN-NamMinhNeural", - "cy-GB-AledNeural", - "cy-GB-NiaNeural", - "zu-ZA-ThandoNeural", - "zu-ZA-ThembaNeural" - ], - "google_tts_voice": [ - "af", - "am", - "ar", - "bg", - "bn", - "bs", - "ca", - "cs", - "cy", - "da", - "de", - "el", - "en", - "es", - "et", - "eu", - "fi", - "fr", - "fr-CA", - "gl", - "gu", - "ha", - "hi", - "hr", - "hu", - "id", - "is", - "it", - "iw", - "ja", - "jw", - "km", - "kn", - "ko", - "la", - "lt", - "lv", - "ml", - "mr", - "ms", - "my", - "ne", - "nl", - "no", - "pa", - "pl", - "pt", - "pt-PT", - "ro", - "ru", - "si", - "sk", - "sq", - "sr", - "su", - "sv", - "sw", - "ta", - "te", - "th", - "tl", - "tr", - "uk", - "ur", - "vi", - "yue", - "zh-CN", - "zh-TW", - "zh" - ], - "whisper_languages": [ - "en", - "zh", - "de", - "es", - "ru", - "ko", - "fr", - "ja", - "pt", - "tr", - "pl", - "ca", - "nl", - "ar", - "sv", - "it", - "id", - "hi", - "fi", - "vi", - "he", - "uk", - "el", - "ms", - "cs", - "ro", - "da", - "hu", - "ta", - "no", - "th", - "ur", - "hr", - "bg", - "lt", - "la", - "mi", - "ml", - "cy", - "sk", - "te", - "fa", - "lv", - "bn", - "sr", - "az", - "sl", - "kn", - "et", - "mk", - "br", - "eu", - "is", - "hy", - "ne", - "mn", - "bs", - "kk", - "sq", - "sw", - "gl", - "mr", - "pa", - "si", - "km", - "sn", - "yo", - "so", - "af", - "oc", - "ka", - "be", - "tg", - "sd", - "gu", - "am", - "yi", - "lo", - "uz", - "fo", - "ht", - "ps", - "tk", - "nn", - "mt", - "sa", - "lb", - "my", - "bo", - "tl", - "mg", - "as", - "tt", - "haw", - "ln", - "ha", - "ba", - "jw", - "su", - "yue" - ], - "fp16": false, - "editing_tab": true, - "inference_tab": true, - "create_and_training_tab": true, - "extra_tab": true, - "separator_tab": true, - "convert_tab": true, - "convert_with_vad_tab": true, - "tts_tab": true, - "effects_tab": true, - "quirk_tab": true, - "create_dataset_tab": true, - "training_tab": true, - "fushion_tab": true, - "read_tab": true, - "onnx_tab": true, - "svc_tab": true, - "downloads_tab": true, - "f0_extractor_tab": true, - "settings_tab": true, - "create_srt_tab": true, - "realtime_tab": true, - "create_reference_tab": true, - "analyzer_tab": true, - "font": "https://fonts.googleapis.com/css2?family=Saira&display=swap", - "app_port": 7860, - "tensorboard_port": 6870, - "num_of_restart": 5, - "server_name": "0.0.0.0", - "app_show_error": true, - "delete_exists_file": false, - "audio_effects_path": "main/inference/audio_effects.py", - "convert_path": "main/inference/conversion/convert.py", - "separate_path": "main/inference/separate_music.py", - "create_dataset_path": "main/inference/create_dataset.py", - "preprocess_path": "main/inference/preprocess/preprocess.py", - "extract_path": "main/inference/extracting/extract.py", - "create_index_path": "main/inference/create_index.py", - "train_path": "main/inference/training/train.py", - "create_reference_path": "main/inference/create_reference.py", - "ico_path": "assets/ico.png", - "csv_path": "assets/spreadsheet.csv", - "weights_path": "assets/weights", - "logs_path": "assets/logs", - "binary_path": "assets/binary", - "f0_path": "assets/f0", - "language_path": "assets/languages", - "presets_path": "assets/presets", - "embedders_path": "assets/models/embedders", - "predictors_path": "assets/models/predictors", - "pretrained_custom_path": "assets/models/pretrained_custom", - "pretrained_v1_path": "assets/models/pretrained_v1", - "pretrained_v2_path": "assets/models/pretrained_v2", - "speaker_diarization_path": "assets/models/speaker_diarization", - "uvr5_path": "assets/models/uvr5", - "audios_path": "audios", - "reference_path": "assets/logs/reference", - "demucs_segments_enable": true, - "demucs_cpu_mode": false, - "limit_f0": 12, - "debug_mode": true, - "pretrain_strict": true, - "cpu_mode": false, - "brain": false, - "tf32": false, - "discord_presence": true, - "f0_min": 50, - "f0_max": 1100, - "turn_on_resize_f0_for_all": false, - "allow_fp16_all_backend": false, - "asio_enabled": false, - "ffmpeg_download": true, - "compile_all": false, - "compile_mode": null, - "compile_cache_dir": "assets/cache", - "check_data": false, - "tensorrt": false, - "faiss_cpu": false, - "allow_unsafe_fairseq": false, - "del4stem": true, - "enable_rmvpe_chunk": false, - "rmvpe_chunk_size": 8000 -} \ No newline at end of file diff --git a/main/configs/config.py b/main/configs/config.py deleted file mode 100644 index ce918e4719a91927e222b323fa29f379fc43f8c5..0000000000000000000000000000000000000000 --- a/main/configs/config.py +++ /dev/null @@ -1,192 +0,0 @@ -import os -import sys -import json -import torch -import onnxruntime - -sys.path.append(os.getcwd()) - -from main.library.backends import directml, opencl, xpu, zluda - -version_config_paths = [ - os.path.join(version, size) - for version in ["v1", "v2"] - for size in ["24000.json", "32000.json", "40000.json", "44100.json", "48000.json"] -] - -def singleton(cls): - instances = {} - - def get_instance(*args, **kwargs): - if cls not in instances: instances[cls] = cls(*args, **kwargs) - return instances[cls] - - return get_instance - -@singleton -class Config: - def __init__(self): - self.configs_path = os.path.join("main", "configs", "config.json") - self.configs = json.load(open(self.configs_path, "r")) - - self.cpu_mode = self.configs.get("cpu_mode", False) - self.cuda_available = torch.cuda.is_available() - self.is_zluda = self.cuda_available and torch.cuda.get_device_name().endswith("[ZLUDA]") - self.hip_version = torch.version.hip if hasattr(torch.version, "hip") else None - if self.is_zluda: zluda.init_zluda() - - self.debug_mode = self.configs.get("debug_mode", False) - self.json_config = self.load_config_json() - self.translations = self.multi_language() - - self.gpu_mem = None - self.per_preprocess = 3.7 - self.device = self.get_default_device() - self.tensorrt = self.configs.get("tensorrt", False) - self.providers = self.get_providers() - - self.cuda_tf32 = self.cuda_available and hasattr(torch.cuda, "is_tf32_supported") and torch.cuda.is_tf32_supported() - self.cuda_bf16 = self.cuda_available and hasattr(torch.cuda, "is_bf16_supported") and torch.cuda.is_bf16_supported() - self.xpu_fp16 = hasattr(torch, "xpu") and torch.xpu.is_available() and hasattr(torch.xpu, "is_bf16_supported") and torch.xpu.is_bf16_supported() - - self.tf32_support = self.device.startswith("cuda") and self.cuda_tf32 - self.bf16_support = self.device.startswith(("cuda", "xpu")) and self.cuda_bf16 or self.xpu_fp16 - - self.brain = self.configs.get("brain", False) and self.bf16_support and not self.cpu_mode - self.tf32 = self.configs.get("tf32", False) and self.tf32_support and not self.is_zluda and not self.cpu_mode - - self.is_half = self.is_fp16() - self.compile_all = self.device.startswith("cuda") and self.configs.get("compile_all", False) - self.compile_mode = self.configs.get("compile_mode", None) - self.allow_is_half = not self.device.startswith(("cpu", "mps", "ocl", "privateuseone")) or self.configs.get("allow_fp16_all_backend", False) - self.x_pad, self.x_query, self.x_center, self.x_max = self.device_config() - if self.compile_all: self.setup_compile() - - def multi_language(self): - try: - lang = self.configs.get("language", "vi-VN") - - if len([l for l in os.listdir(self.configs["language_path"]) if l.endswith(".json")]) < 1: - raise FileNotFoundError("Không tìm thấy bất cứ gói ngôn ngữ nào(No package languages found)") - - if not lang: lang = "vi-VN" - - if lang not in self.configs["support_language"]: - raise ValueError("Ngôn ngữ không được hỗ trợ (Language not supported)") - - lang_path = os.path.join(self.configs["language_path"], f"{lang}.json") - if not os.path.exists(lang_path): - lang_path = os.path.join(self.configs["language_path"], "vi-VN.json") - - with open(lang_path, encoding="utf-8") as f: - translations = json.load(f) - except json.JSONDecodeError: - print(self.translations["empty_json"].format(file=lang)) - pass - - return translations - - def is_fp16(self): - fp16 = self.configs.get("fp16", False) - - if self.device in ["cpu", "mps", "ocl", "privateuseone"] and fp16: - self.configs["fp16"] = False - fp16 = False - - with open(self.configs_path, "w") as f: - json.dump(self.configs, f, indent=4) - - if not fp16: self.per_preprocess = 3.0 - return fp16 - - def load_config_json(self): - configs = {} - - for config_file in version_config_paths: - try: - with open(os.path.join("main", "configs", config_file), "r") as f: - configs[config_file] = json.load(f) - except json.JSONDecodeError: - print(self.translations["empty_json"].format(file=config_file)) - pass - - return configs - - def setup_compile(self): - import importlib - - try: - importlib.import_module("triton") - except ModuleNotFoundError: - self.compile_all = False - self.configs["compile_all"] = False - - with open(self.configs_path, "w") as f: - json.dump(self.configs, f, indent=4) - - if self.compile_all: - dirs = self.configs.get("compile_cache_dir", "none") - - if dirs != "none": - if not os.path.exists(dirs): os.makedirs(dirs) - os.environ["TORCHINDUCTOR_CACHE_DIR"] = dirs - - import torch._inductor.config as config - - torch._dynamo.config.capture_scalar_outputs = True - config.freezing = True - - def device_config(self): - if self.gpu_mem is not None and self.gpu_mem <= 4: - self.per_preprocess = 3.0 - return 1, 5, 30, 32 - - return (3, 10, 60, 65) if self.is_half else (1, 6, 38, 41) - - def get_default_device(self): - if not self.cpu_mode: - if torch.cuda.is_available(): - device = "cuda:0" - self.gpu_mem = torch.cuda.get_device_properties(int(device.split(":")[-1])).total_memory // (1024**3) - elif hasattr(torch, "xpu") and torch.xpu.is_available(): - device = "xpu:0" - self.gpu_mem = torch.xpu.get_device_properties(int(device.split(":")[-1])).total_memory // (1024**3) - xpu.setup_gradscaler() - elif directml.is_available(): - device = "privateuseone:0" - elif opencl.is_available(): - device = "ocl:0" - elif torch.backends.mps.is_available(): - device = "mps" - else: - device = "cpu" - else: - torch.cuda.is_available = lambda : False - directml.is_available = lambda : False - opencl.is_available = lambda : False - torch.backends.mps.is_available = lambda : False - if hasattr(torch, "xpu"): torch.xpu.is_available = lambda: False - - device = "cpu" - - return device - - def get_providers(self): - ort_providers = onnxruntime.get_available_providers() - providers = [] - - if self.device.startswith("cuda"): - if "TensorrtExecutionProvider" in ort_providers and self.tensorrt: providers.append("TensorrtExecutionProvider") - if "CUDAExecutionProvider" in ort_providers: providers.append("CUDAExecutionProvider") - elif "ROCMExecutionProvider" in ort_providers: providers.append("ROCMExecutionProvider") - elif "MIGraphXExecutionProvider" in ort_providers: providers.append("MIGraphXExecutionProvider") - elif "DmlExecutionProvider" in ort_providers: providers.append("DmlExecutionProvider") - elif self.device.startswith("xpu"): - if "OpenVINOExecutionProvider" in ort_providers: providers.append("OpenVINOExecutionProvider") - elif "DnnlExecutionProvider" in ort_providers: providers.append("DnnlExecutionProvider") - elif "DmlExecutionProvider" in ort_providers: providers.append("DmlExecutionProvider") - elif "DmlExecutionProvider" in ort_providers and self.device.startswith(("ocl", "privateuseone")): providers.append("DmlExecutionProvider") - elif "CoreMLExecutionProvider" in ort_providers and self.device.startswith("mps"): providers.append("CoreMLExecutionProvider") - - providers.append("CPUExecutionProvider") - return providers \ No newline at end of file diff --git a/main/configs/rpc.py b/main/configs/rpc.py deleted file mode 100644 index b23bb867fc7cb64583c0c7f0f5a175971a48665f..0000000000000000000000000000000000000000 --- a/main/configs/rpc.py +++ /dev/null @@ -1,81 +0,0 @@ -import os -import sys -import json -import time -import struct -import codecs - -sys.path.append(os.getcwd()) - -from main.app.variables import translations - -CLIENT_ID = "1392816674159202396" - -def create_payload(opcode, payload): - data = json.dumps(payload).encode("utf-8") - - return struct.pack( - "= 1: - b, a = butter(4, frequency / (0.5 * sample_rate), btype='low') - boosted = _filtfilt(b, a, audio) - return boosted * (10 ** (gain_db / 20)) - return audio - - def treble_boost(audio, gain_db, frequency, sample_rate): - if gain_db >= 1: - b, a = butter(4, frequency / (0.5 * sample_rate), btype='high') - boosted = _filtfilt(b, a, audio) - return boosted * (10 ** (gain_db / 20)) - return audio - - def fade_out_effect(audio, sr, duration=3.0): - length = int(duration * sr) - end = audio.shape[0] - if length > end: length = end - start = end - length - audio[start:end] = audio[start:end] * np.linspace(1.0, 0.0, length) - return audio - - def fade_in_effect(audio, sr, duration=3.0): - length = int(duration * sr) - start = 0 - if length > audio.shape[0]: length = audio.shape[0] - end = length - audio[start:end] = audio[start:end] * np.linspace(0.0, 1.0, length) - return audio - - if not input_path or not os.path.exists(input_path): - logger.warning(translations["input_not_valid"]) - sys.exit(1) - - if not output_path: - logger.warning(translations["output_not_valid"]) - sys.exit(1) - - if os.path.exists(output_path): os.remove(output_path) - - try: - input_path = input_path.strip(" ").strip('"').strip("\n").strip('"').strip(" ") - try: - audio, sample_rate = sf.read(input_path, dtype=np.float32) - except: - audio, sample_rate = librosa.load(input_path, sr=None) - except Exception as e: - logger.debug(f"{translations['errors_loading_audio']}: {e}") - raise RuntimeError(f"{translations['errors_loading_audio']}: {e}") - - try: - board = Pedalboard([HighpassFilter()]) - - if chorus: - board.append( - Chorus( - depth=chorus_depth, - rate_hz=chorus_rate, - mix=chorus_mix, - centre_delay_ms=chorus_delay, - feedback=chorus_feedback - ) - ) - - if distortion: - board.append( - Distortion( - drive_db=distortion_drive - ) - ) - - if reverb: - board.append( - Reverb( - room_size=reverb_room_size, - damping=reverb_damping, - wet_level=reverb_wet_level, - dry_level=reverb_dry_level, - width=reverb_width, - freeze_mode=int(reverb_freeze_mode) - ) - ) - - if pitchshift: - board.append( - PitchShift( - semitones=pitch_shift - ) - ) - - if delay: - board.append( - Delay( - delay_seconds=delay_seconds, - feedback=delay_feedback, - mix=delay_mix - ) - ) - - if compressor: - board.append( - Compressor( - threshold_db=compressor_threshold, - ratio=compressor_ratio, - attack_ms=compressor_attack_ms, - release_ms=compressor_release_ms - ) - ) - - if limiter: - board.append( - Limiter( - threshold_db=limiter_threshold, - release_ms=limiter_release - ) - ) - - if gain: - board.append( - Gain( - gain_db=gain_db - ) - ) - - if bitcrush: - board.append( - Bitcrush( - bit_depth=bitcrush_bit_depth - ) - ) - - if clipping: - board.append( - Clipping( - threshold_db=clipping_threshold - ) - ) - - if phaser: board.append( - Phaser( - rate_hz=phaser_rate_hz, - depth=phaser_depth, - centre_frequency_hz=phaser_centre_frequency_hz, - feedback=phaser_feedback, - mix=phaser_mix - ) - ) - - processed_audio = board(audio, sample_rate) - - if treble_bass_boost: - processed_audio = bass_boost( - processed_audio, - bass_boost_db, - bass_boost_frequency, - sample_rate - ) - - processed_audio = treble_boost( - processed_audio, - treble_boost_db, - treble_boost_frequency, - sample_rate - ) - - if fade_in_out: - processed_audio = fade_in_effect( - processed_audio, - sample_rate, - fade_in_duration - ) - - processed_audio = fade_out_effect( - processed_audio, - sample_rate, - fade_out_duration - ) - - if resample and resample_sr != sample_rate and resample_sr > 0: - processed_audio = librosa.resample( - processed_audio, - orig_sr=sample_rate, - target_sr=resample_sr, - res_type="soxr_vhq" - ) - - sample_rate = resample_sr - - sf.write( - replace_export_format(output_path, export_format), - processed_audio, - sample_rate, - format=export_format - ) - - if audio_combination: - pydub_load( - audio_combination_input, - combination_volume - ).overlay( - pydub_load( - replace_export_format(output_path, export_format), - main_volume - ) - ).export( - replace_export_format(output_path, export_format), - format=export_format - ) - except Exception as e: - import traceback - logger.debug(traceback.format_exc()) - raise RuntimeError(translations["apply_error"].format(e=e)) - return output_path - -def main(): - args = parse_arguments() - - process_audio( - input_path=args.input_path, - output_path=args.output_path, - resample=args.resample, - resample_sr=args.resample_sr, - chorus_depth=args.chorus_depth, - chorus_rate=args.chorus_rate, - chorus_mix=args.chorus_mix, - chorus_delay=args.chorus_delay, - chorus_feedback=args.chorus_feedback, - distortion_drive=args.drive_db, - reverb_room_size=args.reverb_room_size, - reverb_damping=args.reverb_damping, - reverb_wet_level=args.reverb_wet_level, - reverb_dry_level=args.reverb_dry_level, - reverb_width=args.reverb_width, - reverb_freeze_mode=args.reverb_freeze_mode, - pitch_shift=args.pitch_shift, - delay_seconds=args.delay_seconds, - delay_feedback=args.delay_feedback, - delay_mix=args.delay_mix, - compressor_threshold=args.compressor_threshold, - compressor_ratio=args.compressor_ratio, - compressor_attack_ms=args.compressor_attack_ms, - compressor_release_ms=args.compressor_release_ms, - limiter_threshold=args.limiter_threshold, - limiter_release=args.limiter_release, - gain_db=args.gain_db, - bitcrush_bit_depth=args.bitcrush_bit_depth, - clipping_threshold=args.clipping_threshold, - phaser_rate_hz=args.phaser_rate_hz, - phaser_depth=args.phaser_depth, - phaser_centre_frequency_hz=args.phaser_centre_frequency_hz, - phaser_feedback=args.phaser_feedback, - phaser_mix=args.phaser_mix, - bass_boost_db=args.bass_boost_db, - bass_boost_frequency=args.bass_boost_frequency, - treble_boost_db=args.treble_boost_db, - treble_boost_frequency=args.treble_boost_frequency, - fade_in_duration=args.fade_in_duration, - fade_out_duration=args.fade_out_duration, - export_format=args.export_format, - chorus=args.chorus, - distortion=args.distortion, - reverb=args.reverb, - pitchshift=args.pitchshift, - delay=args.delay, - compressor=args.compressor, - limiter=args.limiter, - gain=args.gain, - bitcrush=args.bitcrush, - clipping=args.clipping, - phaser=args.phaser, - treble_bass_boost=args.treble_bass_boost, - fade_in_out=args.fade_in_out, - audio_combination=args.audio_combination, - audio_combination_input=args.audio_combination_input, - main_volume=args.main_volume, - combination_volume=args.combination_volume - ) - -if __name__ == "__main__": main() \ No newline at end of file diff --git a/main/inference/conversion/convert.py b/main/inference/conversion/convert.py deleted file mode 100644 index f7e29c62303f749257d5fe7a162465806b7a37b8..0000000000000000000000000000000000000000 --- a/main/inference/conversion/convert.py +++ /dev/null @@ -1,711 +0,0 @@ -import os -import sys -import time -import torch -import spaces -import librosa -import logging -import argparse -import warnings - -import numpy as np -import soundfile as sf - -from tqdm import tqdm -from scipy import signal - -sys.path.append(os.getcwd()) - -from main.app.core.ui import replace_export_format -from main.inference.conversion.pipeline import Pipeline -from main.library.audio.audio import load_audio, cut, restore -from main.app.variables import config, logger, translations, file_types -from main.library.audio.audio_processing import preprocess, postprocess -from main.library.utils import check_assets, check_upscaler, load_embedders_model, clear_gpu_cache, load_model, strtobool, load_faiss_index - -if not config.debug_mode: - warnings.filterwarnings("ignore") - - for l in [ - "torch", - "faiss", - "omegaconf", - "httpx", - "httpcore", - "faiss.loader", - "numba.core", - "urllib3", - "transformers", - "matplotlib" - ]: - logging.getLogger(l).setLevel(logging.ERROR) - -def parse_arguments(): - parser = argparse.ArgumentParser() - parser.add_argument("--convert", action='store_true') - parser.add_argument("--pitch", type=int, default=0) - parser.add_argument("--filter_radius", type=int, default=3) - parser.add_argument("--index_rate", type=float, default=0.5) - parser.add_argument("--rms_mix_rate", type=float, default=1) - parser.add_argument("--protect", type=float, default=0.33) - parser.add_argument("--hop_length", type=int, default=64) - parser.add_argument("--f0_method", type=str, default="rmvpe") - parser.add_argument("--embedder_model", type=str, default="hubert_base") - parser.add_argument("--input_path", type=str, required=True) - parser.add_argument("--output_path", type=str, default="./audios/output.wav") - parser.add_argument("--export_format", type=str, default="wav") - parser.add_argument("--pth_path", type=str, required=True) - parser.add_argument("--index_path", type=str, default="") - parser.add_argument("--f0_autotune", type=lambda x: bool(strtobool(x)), default=False) - parser.add_argument("--f0_autotune_strength", type=float, default=1) - parser.add_argument("--clean_audio", type=lambda x: bool(strtobool(x)), default=False) - parser.add_argument("--clean_strength", type=float, default=0.7) - parser.add_argument("--resample_sr", type=int, default=0) - parser.add_argument("--split_audio", type=lambda x: bool(strtobool(x)), default=False) - parser.add_argument("--checkpointing", type=lambda x: bool(strtobool(x)), default=False) - parser.add_argument("--f0_file", type=str, default="") - parser.add_argument("--predictor_onnx", type=lambda x: bool(strtobool(x)), default=False) - parser.add_argument("--embedders_mode", type=str, default="fairseq") - parser.add_argument("--formant_shifting", type=lambda x: bool(strtobool(x)), default=False) - parser.add_argument("--formant_qfrency", type=float, default=0.8) - parser.add_argument("--formant_timbre", type=float, default=0.8) - parser.add_argument("--proposal_pitch", type=lambda x: bool(strtobool(x)), default=False) - parser.add_argument("--proposal_pitch_threshold", type=float, default=255.0) - parser.add_argument("--audio_processing", type=lambda x: bool(strtobool(x)), default=False) - parser.add_argument("--alpha", type=float, default=0.5) - parser.add_argument("--sid", type=int, default=0, required=False) - parser.add_argument("--embedders_mix", type=lambda x: bool(strtobool(x)), default=False) - parser.add_argument("--embedders_mix_layers", type=int, default=9, required=False) - parser.add_argument("--embedders_mix_ratio", type=float, default=0.5) - parser.add_argument("--noise_scale", type=float, default=0.35) - parser.add_argument("--nprobe", type=int, default=1) - parser.add_argument("--audio_upscaler", type=lambda x: bool(strtobool(x)), default=False) - - return parser.parse_args() - -def main(): - args = parse_arguments() - - ( - pitch, - filter_radius, - index_rate, - rms_mix_rate, - protect, - hop_length, - f0_method, - input_path, - output_path, - pth_path, - index_path, - f0_autotune, - f0_autotune_strength, - clean_audio, - clean_strength, - export_format, - embedder_model, - resample_sr, - split_audio, - checkpointing, - f0_file, - predictor_onnx, - embedders_mode, - formant_shifting, - formant_qfrency, - formant_timbre, - proposal_pitch, - proposal_pitch_threshold, - audio_processing, - alpha, - sid, - embedders_mix, - embedders_mix_layers, - embedders_mix_ratio, - noise_scale, - nprobe, - audio_upscaler - ) = ( - args.pitch, - args.filter_radius, - args.index_rate, - args.rms_mix_rate, - args.protect, - args.hop_length, - args.f0_method, - args.input_path, - args.output_path, - args.pth_path, - args.index_path, - args.f0_autotune, - args.f0_autotune_strength, - args.clean_audio, - args.clean_strength, - args.export_format, - args.embedder_model, - args.resample_sr, - args.split_audio, - args.checkpointing, - args.f0_file, - args.predictor_onnx, - args.embedders_mode, - args.formant_shifting, - args.formant_qfrency, - args.formant_timbre, - args.proposal_pitch, - args.proposal_pitch_threshold, - args.audio_processing, - args.alpha, - args.sid, - args.embedders_mix, - args.embedders_mix_layers, - args.embedders_mix_ratio, - args.noise_scale, - args.nprobe, - args.audio_upscaler - ) - - run_convert_script( - pitch=pitch, - filter_radius=filter_radius, - index_rate=index_rate, - rms_mix_rate=rms_mix_rate, - protect=protect, - hop_length=hop_length, - f0_method=f0_method, - input_path=input_path, - output_path=output_path, - pth_path=pth_path, - index_path=index_path, - f0_autotune=f0_autotune, - f0_autotune_strength=f0_autotune_strength, - clean_audio=clean_audio, - clean_strength=clean_strength, - export_format=export_format, - embedder_model=embedder_model, - resample_sr=resample_sr, - split_audio=split_audio, - checkpointing=checkpointing, - f0_file=f0_file, - predictor_onnx=predictor_onnx, - embedders_mode=embedders_mode, - formant_shifting=formant_shifting, - formant_qfrency=formant_qfrency, - formant_timbre=formant_timbre, - proposal_pitch=proposal_pitch, - proposal_pitch_threshold=proposal_pitch_threshold, - audio_processing=audio_processing, - alpha=alpha, - sid=sid, - embedders_mix=embedders_mix, - embedders_mix_layers=embedders_mix_layers, - embedders_mix_ratio=embedders_mix_ratio, - noise_scale=noise_scale, - nprobe=nprobe, - audio_upscaler=audio_upscaler - ) - -@spaces.GPU -def run_convert_script( - pitch=0, - filter_radius=3, - index_rate=0.5, - rms_mix_rate=1, - protect=0.5, - hop_length=64, - f0_method="rmvpe", - input_path=None, - output_path="./output.wav", - pth_path=None, - index_path=None, - f0_autotune=False, - f0_autotune_strength=1, - clean_audio=False, - clean_strength=0.7, - export_format="wav", - embedder_model="hubert_base", - resample_sr=0, - split_audio=False, - checkpointing=False, - f0_file=None, - predictor_onnx=False, - embedders_mode="fairseq", - formant_shifting=False, - formant_qfrency=0.8, - formant_timbre=0.8, - proposal_pitch=False, - proposal_pitch_threshold=255.0, - audio_processing=False, - alpha=0.5, - sid=0, - embedders_mix = False, - embedders_mix_layers = 9, - embedders_mix_ratio = 0.5, - noise_scale = 0.35, - nprobe = 1, - audio_upscaler = False -): - if audio_upscaler: check_upscaler() - check_assets( - f0_method, - embedder_model, - predictor_onnx=predictor_onnx, - embedders_mode=embedders_mode - ) - - log_data = { - translations["pitch"]: pitch, - translations["filter_radius"]: filter_radius, - translations["index_strength"]: index_rate, - translations["rms_mix_rate"]: rms_mix_rate, - translations["protect"]: protect, - translations["hop_length"]: hop_length, - translations["f0_method"]: f0_method, - translations["audio_path"]: input_path, - translations["output_path"]: replace_export_format(output_path, export_format), - translations["model_path"]: pth_path, - translations["indexpath"]: index_path, - translations["autotune"]: f0_autotune, - translations["autotune_rate_info"]: f0_autotune_strength, - translations["clear_audio"]: clean_audio, - translations["clean_strength"]: clean_strength, - translations["sample_rate"]: resample_sr, - translations["export_format"]: export_format, - translations["hubert_model"]: embedder_model, - translations["split_audio"]: split_audio, - translations["memory_efficient_training"]: checkpointing, - translations["predictor_onnx"]: predictor_onnx, - translations["f0_file"]: f0_file, - translations["embed_mode"]: embedders_mode, - translations["proposal_pitch"]: proposal_pitch, - translations["proposal_pitch_threshold"]: proposal_pitch_threshold, - translations["audio_processing"]: audio_processing, - translations["alpha_label"]: alpha, - translations["embedders_mix"]: embedders_mix, - translations["embedders_mix_layers"]: embedders_mix_layers, - translations["embedders_mix_ratio"]: embedders_mix_ratio, - translations["formant_qfrency"]: formant_qfrency, - translations["formant_timbre"]: formant_timbre, - translations["noise_scale"]: noise_scale, - translations["nprobe"]: nprobe, - translations["audio_upscaler"]: audio_upscaler - } - - for key, value in log_data.items(): - logger.debug(f"{key}: {value}") - - if not pth_path or not os.path.exists(pth_path) or os.path.isdir(pth_path) or not pth_path.endswith((".pth", ".onnx")): - logger.warning(translations["provide_file"].format(filename=translations["model"])) - sys.exit(1) - - cvt = VoiceConverter(pth_path, sid, noise_scale, checkpointing) - - pid_path = os.path.join("assets", "convert_pid.txt") - with open(pid_path, "w") as pid_file: - pid_file.write(str(os.getpid())) - - def convert_audio(audio_path, output_audio): - cvt.convert_audio( - pitch=pitch, - filter_radius=filter_radius, - index_rate=index_rate, - rms_mix_rate=rms_mix_rate, - protect=protect, - hop_length=hop_length, - f0_method=f0_method, - audio_input_path=audio_path, - audio_output_path=output_audio, - index_path=index_path, - f0_autotune=f0_autotune, - f0_autotune_strength=f0_autotune_strength, - clean_audio=clean_audio, - clean_strength=clean_strength, - export_format=export_format, - embedder_model=embedder_model, - resample_sr=resample_sr, - f0_file=f0_file, - predictor_onnx=predictor_onnx, - embedders_mode=embedders_mode, - formant_shifting=formant_shifting, - formant_qfrency=formant_qfrency, - formant_timbre=formant_timbre, - split_audio=split_audio, - proposal_pitch=proposal_pitch, - proposal_pitch_threshold=proposal_pitch_threshold, - audio_processing=audio_processing, - alpha=alpha, - embedders_mix=embedders_mix, - embedders_mix_layers=embedders_mix_layers, - embedders_mix_ratio=embedders_mix_ratio, - nprobe=nprobe, - audio_upscaler=audio_upscaler - ) - - start_time = time.time() - - if os.path.isdir(input_path): - logger.info(translations["convert_batch"]) - - audio_files = [ - f - for f in os.listdir(input_path) - if f.lower().endswith(tuple(file_types)) - ] - - if not audio_files: - logger.warning(translations["not_found_audio"]) - sys.exit(1) - - logger.info(translations["found_audio"].format(audio_files=len(audio_files))) - - for audio in audio_files: - audio_path = os.path.join(input_path, audio) - output_audio = os.path.join(input_path, os.path.splitext(audio)[0] + f"_output.{export_format}") - - logger.info(f"{translations['convert_audio']} '{audio_path}'...") - if os.path.exists(output_audio): os.remove(output_audio) - - convert_audio(audio_path, output_audio) - - logger.info( - translations["convert_batch_success"].format( - elapsed_time=f"{(time.time() - start_time):.2f}", - output_path=replace_export_format(output_path, export_format) - ) - ) - else: - if not os.path.exists(input_path): - logger.warning(translations["not_found_audio"]) - sys.exit(1) - - logger.info(f"{translations['convert_audio']} '{input_path}'...") - if os.path.exists(output_path): os.remove(output_path) - - convert_audio(input_path, output_path) - - logger.info( - translations["convert_audio_success"].format( - input_path=input_path, - elapsed_time=f"{(time.time() - start_time):.2f}", - output_path=replace_export_format(output_path, export_format) - ) - ) - - if os.path.exists(pid_path): os.remove(pid_path) - -class VoiceConverter: - def __init__( - self, - model_path, - sid = 0, - noise_scale = 0.35, - checkpointing = False - ): - self.config = config - self.device = config.device - self.hubert_model = None - self.tgt_sr = None - self.net_g = None - self.vc = None - self.cpt = None - self.version = None - self.n_spk = None - self.use_f0 = None - self.loaded_model = None - self.index = None - self.big_tsr = None - self.vocoder = "Default" - self.checkpointing = checkpointing - self.sample_rate = 16000 - self.sid = sid - self.noise_scale = noise_scale - self.get_vc(model_path, sid) - - def convert_audio( - self, - audio_input_path, - audio_output_path, - index_path, - embedder_model, - pitch, - f0_method, - index_rate, - rms_mix_rate, - protect, - hop_length, - f0_autotune, - f0_autotune_strength, - filter_radius, - clean_audio, - clean_strength, - export_format, - resample_sr = 0, - f0_file = None, - predictor_onnx = False, - embedders_mode = "fairseq", - formant_shifting = False, - formant_qfrency = 0.8, - formant_timbre = 0.8, - split_audio = False, - proposal_pitch = False, - proposal_pitch_threshold = 0, - audio_processing = False, - alpha = 0.5, - embedders_mix = False, - embedders_mix_layers = 9, - embedders_mix_ratio = 0.5, - nprobe = 1, - audio_upscaler = False - ): - try: - with tqdm(total=10, desc=translations["convert_audio"], ncols=100, unit="a", leave=not split_audio) as pbar: - audio = load_audio( - audio_input_path, - sample_rate=self.sample_rate, - formant_shifting=formant_shifting, - formant_qfrency=formant_qfrency, - formant_timbre=formant_timbre - ) - - if audio_processing: - audio = preprocess( - audio, - self.sample_rate - ) - - try: - audio_max = np.abs(audio).max() / 0.95 - if audio_max > 1: audio /= audio_max - except: - import shutil - - shutil.copy(audio_input_path, audio_output_path) - return - - if self.hubert_model is None: - models = load_embedders_model(embedder_model, embedders_mode) - if isinstance(models, torch.nn.Module): - models = models.to(self.device).to(torch.float16 if self.config.is_half else torch.float32).eval() - if config.compile_all and embedders_mode != "whisper": models = torch.compile(models, mode=self.config.compile_mode) - self.hubert_model = models - - if index_rate != 0 and (self.index is None or self.big_tsr is None): - self.index, self.big_tsr = load_faiss_index(index_path.strip().strip('"').strip("\n").strip('"').strip().replace("trained", "added"), nprobe) - - pbar.update(1) - if split_audio: - pbar.close() - - chunks = cut( - audio, - self.sample_rate, - db_thresh=-60, - min_interval=500 - ) - - logger.info(f"{translations['split_total']}: {len(chunks)}") - pbar = tqdm(total=len(chunks) * 5 + 4, desc=translations["convert_audio"], ncols=100, unit="a", leave=True) - else: chunks = [(audio, 0, 0)] - - pbar.update(1) - converted_chunks = [ - ( - start, - end, - self.vc.pipeline( - logger=logger, - model=self.hubert_model, - net_g=self.net_g, - sid=self.sid, - audio=waveform, - f0_up_key=pitch, - f0_method=f0_method, - index=self.index, - big_tsr=self.big_tsr, - index_rate=index_rate, - pitch_guidance=self.use_f0, - filter_radius=filter_radius, - rms_mix_rate=rms_mix_rate, - version=self.version, - protect=protect, - hop_length=hop_length, - f0_autotune=f0_autotune, - f0_autotune_strength=f0_autotune_strength, - f0_file=f0_file, - predictor_onnx=predictor_onnx, - pbar=pbar, - proposal_pitch=proposal_pitch, - proposal_pitch_threshold=proposal_pitch_threshold, - energy_use=self.energy, - alpha=alpha, - embedders_mix=embedders_mix, - embedders_mix_layers=embedders_mix_layers, - embedders_mix_ratio=embedders_mix_ratio, - ) - ) - for waveform, start, end in chunks - ] - - pbar.update(1) - - audio_output = restore( - converted_chunks, - total_len=len(audio), - dtype=converted_chunks[0][2].dtype - ) if split_audio else converted_chunks[0][2] - - if audio_processing: - audio_output = postprocess( - audio_output, - self.tgt_sr - ) - - if clean_audio: - from main.library.audio.noisereduce import TorchGate - - if not hasattr(self, "tg"): - self.tg = TorchGate( - self.tgt_sr, - prop_decrease=clean_strength - ).to(self.device) - - audio_output = self.tg( - torch.from_numpy(audio_output).unsqueeze(0).to(self.device).float() - ).squeeze(0).cpu().detach().numpy() - - target_len = int(np.round(len(audio) / self.sample_rate * self.tgt_sr)) - if len(audio_output) != target_len: audio_output = signal.resample_poly(audio_output, target_len, len(audio_output)) - - audio_output_resample = None - - if audio_upscaler: - from main.library.audio.upscaler import FlashSR, upscaler - - if not hasattr(self, "flash_sr"): - self.flash_sr = FlashSR( - os.path.join("assets", "models", "upscalers", "upscalers.pth"), - device=self.device, - is_half=self.config.is_half - ) - - audio_output_resample = librosa.resample( - audio_output, - orig_sr=self.tgt_sr, - target_sr=192000, - res_type="soxr_vhq" - ) - - audio_output_resample = upscaler(audio_output_resample, self.flash_sr, pbar, device=self.device) - audio_output_resample = audio_output_resample.float().flatten().cpu().detach().numpy() - self.tgt_sr = 192000 - elif self.tgt_sr != resample_sr and resample_sr > 0: - audio_output_resample = librosa.resample( - audio_output, - orig_sr=self.tgt_sr, - target_sr=resample_sr, - res_type="soxr_vhq" - ) - - self.tgt_sr = resample_sr - - pbar.update(1) - - try: - sf.write( - audio_output_path, - audio_output if audio_output_resample is None else audio_output_resample, - self.tgt_sr, - format=export_format - ) - except: - logger.info(translations["sr_not_support"]) - - sf.write( - audio_output_path, - librosa.resample( - audio_output, - orig_sr=self.tgt_sr, - target_sr=48000, - res_type="soxr_vhq" - ), - 48000, - format=export_format - ) - - pbar.update(1) - except Exception as e: - import traceback - - logger.debug(traceback.format_exc()) - logger.error(translations["error_convert"].format(e=e)) - - def get_vc(self, weight_root, sid): - if sid == "" or sid == []: - self.cleanup() - clear_gpu_cache() - - if not self.loaded_model or self.loaded_model != weight_root: - self.loaded_model = weight_root - self.cpt = load_model(weight_root) - if self.cpt is not None: self.setup() - - def cleanup(self): - if self.hubert_model is not None: - del self.net_g, self.n_spk, self.vc, self.hubert_model, self.tgt_sr, self.index, self.big_tsr - self.hubert_model = self.net_g = self.n_spk = self.vc = self.tgt_sr = self.index = self.big_tsr = None - clear_gpu_cache() - - del self.net_g, self.cpt - clear_gpu_cache() - self.cpt = None - - def setup(self): - if self.cpt is not None: - if self.loaded_model.endswith(".pth"): - from main.library.algorithm.synthesizers import Synthesizer, SynthesizerSVC - - self.tgt_sr = self.cpt["config"][-1] - self.cpt["config"][-3] = self.cpt["weight"]["emb_g.weight"].shape[0] - - self.use_f0 = self.cpt.get("f0", 1) - self.version = self.cpt.get("version", "v1") - self.vocoder = self.cpt.get("vocoder", "Default") - self.energy = self.cpt.get("energy", False) - self.architecture = self.cpt.get("architecture", "RVC") - - if self.vocoder != "Default": self.config.is_half = False - - if self.architecture == "RVC": - self.net_g = Synthesizer( - *self.cpt["config"], - use_f0=self.use_f0, - text_enc_hidden_dim=768 if self.version == "v2" else 256, - vocoder=self.vocoder, - checkpointing=self.checkpointing, - energy=self.energy - ) - else: - self.net_g = SynthesizerSVC( - *self.cpt["config"], - text_enc_hidden_dim=768 if self.version == "v2" else 256, - vocoder=self.vocoder, - checkpointing=self.checkpointing, - noise_scale=self.noise_scale - ) - - del self.net_g.enc_q - - self.net_g.load_state_dict(self.cpt["weight"], strict=False) - self.net_g.eval().to(self.device) - self.net_g.to(torch.float16 if self.config.is_half else torch.float32) - if config.compile_all: self.net_g = torch.compile(self.net_g, mode=self.config.compile_mode) - self.n_spk = self.cpt["config"][-3] - else: - self.net_g = self.cpt.to(config.device) - self.tgt_sr = self.cpt.cpt.get("tgt_sr", 32000) - self.use_f0 = self.cpt.cpt.get("f0", 1) - self.version = self.cpt.cpt.get("version", "v1") - self.energy = self.cpt.cpt.get("energy", False) - - self.vc = Pipeline(self.tgt_sr, self.config) - -if __name__ == "__main__": main() \ No newline at end of file diff --git a/main/inference/conversion/pipeline.py b/main/inference/conversion/pipeline.py deleted file mode 100644 index 8d093c63b7029d63e7c7d541e19af8ff109fa2e7..0000000000000000000000000000000000000000 --- a/main/inference/conversion/pipeline.py +++ /dev/null @@ -1,316 +0,0 @@ -import os -import sys -import torch - -import numpy as np -import torch.nn.functional as F - -from scipy import signal - -sys.path.append(os.getcwd()) - -from main.library.audio.features import change_rms -from main.app.variables import translations, configs -from main.library.utils import extract_features, clear_gpu_cache - -bh, ah = signal.butter( - N=5, - Wn=48, - btype="high", - fs=16000 -) - -class Pipeline: - def __init__( - self, - tgt_sr, - config - ): - self.x_pad = config.x_pad - self.x_query = config.x_query - self.x_center = config.x_center - self.x_max = config.x_max - self.sample_rate = 16000 - self.window = 160 - self.t_pad = self.sample_rate * self.x_pad - self.t_pad_tgt = tgt_sr * self.x_pad - self.t_pad2 = self.t_pad * 2 - self.t_query = self.sample_rate * self.x_query - self.t_center = self.sample_rate * self.x_center - self.t_max = self.sample_rate * self.x_max - self.f0_min = configs.get("f0_min", 50) - self.f0_max = configs.get("f0_max", 1100) - self.device = config.device - self.is_half = config.is_half - self.dtype = torch.float16 if self.is_half else torch.float32 - self.tgt_sr = tgt_sr - - def voice_conversion( - self, - model, - net_g, - sid, - audio0, - pitch, - pitchf, - index, - big_tsr, - index_rate, - version, - protect, - energy, - embedders_mix = False, - embedders_mix_layers = 9, - embedders_mix_ratio = 0.5 - ): - pitch_guidance = pitch is not None and pitchf is not None - energy_use = energy is not None - - feats = torch.from_numpy(audio0).to(self.device).to(self.dtype) - feats = feats.mean(-1) if feats.dim() == 2 else feats - assert feats.dim() == 1, feats.dim() - - with torch.no_grad(): - feats = extract_features( - model, - feats.view(1, -1), - version, - mix=embedders_mix, - mix_layers=embedders_mix_layers, - mix_ratio=embedders_mix_ratio - ) - - feats0 = feats.clone() if protect < 0.5 and pitch_guidance else None - - if index is not None and big_tsr is not None and index_rate != 0: - score, ix = index.search(feats[0], k=8) - weight = (1 / score).square() - query = (big_tsr[ix] * (weight / weight.sum(dim=1, keepdim=True)).unsqueeze(2)).sum(dim=1) - feats = query.unsqueeze(0) * index_rate + (1.0 - index_rate) * feats - - feats = F.interpolate(feats.permute(0, 2, 1), scale_factor=2).permute(0, 2, 1) - p_len = min(audio0.shape[0] // self.window, feats.shape[1]) - - if pitch_guidance: pitch, pitchf = pitch[:, :p_len], pitchf[:, :p_len] - if energy_use: energy = energy[:p_len].unsqueeze(0) - - if feats0 is not None: - pitchff = pitchf.clone() - pitchff[pitchf > 0] = 1 - pitchff[pitchf < 1] = protect - pitchff = pitchff.unsqueeze(-1) - - feats0 = F.interpolate(feats0.permute(0, 2, 1), scale_factor=2).permute(0, 2, 1) - feats = (feats * pitchff + feats0 * (1 - pitchff)).to(feats0.dtype) - - p_len = torch.tensor([p_len], device=self.device).long() - feats = feats.to(self.dtype) - - audio1 = net_g.infer( - feats, - p_len, - pitch if pitch_guidance else None, - pitchf.to(self.dtype) if pitch_guidance else None, - sid, - energy.to(self.dtype) if energy_use else None - )[0][0, 0].data.cpu().float().numpy() - - del feats, feats0, p_len - - clear_gpu_cache() - return audio1 - - def pipeline( - self, - logger, - model, - net_g, - sid, - audio, - f0_up_key, - f0_method, - index, - big_tsr, - index_rate, - pitch_guidance, - filter_radius, - rms_mix_rate, - version, - protect, - hop_length, - f0_autotune, - f0_autotune_strength, - f0_file=None, - predictor_onnx=False, - pbar=None, - proposal_pitch=False, - proposal_pitch_threshold=255.0, - energy_use=False, - alpha = 0.5, - embedders_mix = False, - embedders_mix_layers = 9, - embedders_mix_ratio = 0.5, - ): - opt_ts, audio_opt = [], [] - audio = signal.filtfilt(bh, ah, audio) - audio_pad = np.pad(audio, (self.window // 2, self.window // 2), mode="reflect") - - if audio_pad.shape[0] > self.t_max: - audio_sum = np.zeros_like(audio) - - for i in range(self.window): - audio_sum += audio_pad[i : i - self.window] - - for t in range(self.t_center, audio.shape[0], self.t_center): - opt_ts.append( - t - self.t_query + np.where(np.abs(audio_sum[t - self.t_query : t + self.t_query]) == np.abs(audio_sum[t - self.t_query : t + self.t_query]).min())[0][0] - ) - - pbar.update(1) - - s = 0 - t, inp_f0 = None, None - audio_pad = np.pad(audio, (self.t_pad, self.t_pad), mode="reflect") - sid = torch.tensor(sid, device=self.device).unsqueeze(0).long() - p_len = audio_pad.shape[0] // self.window - - if f0_file and os.path.exists(f0_file) and f0_file.endswith(".txt"): - try: - with open(f0_file, "r") as f: - raw_lines = f.read() - - if len(raw_lines) > 0: - inp_f0 = [] - - for line in raw_lines.strip("\n").split("\n"): - inp_f0.append([float(i) for i in line.split(",")]) - - inp_f0 = np.array(inp_f0, dtype=np.float32) - except: - logger.error(translations["error_readfile"]) - inp_f0 = None - - pbar.update(1) - - if pitch_guidance: - if not hasattr(self, "f0_generator"): - from main.library.predictors.Generator import Generator - - self.f0_generator = Generator( - self.sample_rate, - hop_length, - self.f0_min, - self.f0_max, - alpha, - self.is_half, - self.device, - predictor_onnx - ) - - pitch, pitchf = self.f0_generator.calculator( - self.x_pad, - f0_method, - audio_pad, - f0_up_key, - p_len, - filter_radius, - f0_autotune, - f0_autotune_strength, - manual_f0=inp_f0, - proposal_pitch=proposal_pitch, - proposal_pitch_threshold=proposal_pitch_threshold - ) - - if self.device == "mps": pitchf = pitchf.astype(np.float32) - - pitch, pitchf = ( - torch.tensor(pitch[:p_len], device=self.device).unsqueeze(0).long(), - torch.tensor(pitchf[:p_len], device=self.device).unsqueeze(0).float() - ) - - pbar.update(1) - - if energy_use: - if not hasattr(self, "rms_extract"): - from main.inference.extracting.rms import RMSEnergyExtractor - - self.rms_extract = RMSEnergyExtractor( - frame_length=2048, - hop_length=self.window, - center=True, - pad_mode = "reflect" - ).to(self.device).eval() - - energy = self.rms_extract( - torch.from_numpy(audio_pad).to(self.device).unsqueeze(0) - )[:p_len].to(self.device).float() - - pbar.update(1) - pbar.total = pbar.total + len(opt_ts) - pbar.refresh() - - for t in opt_ts: - t = t // self.window * self.window - audio_opt.append( - self.voice_conversion( - model, - net_g, - sid, - audio_pad[s : t + self.t_pad2 + self.window], - pitch[:, s // self.window : (t + self.t_pad2) // self.window] if pitch_guidance else None, - pitchf[:, s // self.window : (t + self.t_pad2) // self.window] if pitch_guidance else None, - index, - big_tsr, - index_rate, - version, - protect, - energy[:, s // self.window : (t + self.t_pad2) // self.window] if energy_use else None, - embedders_mix=embedders_mix, - embedders_mix_layers=embedders_mix_layers, - embedders_mix_ratio=embedders_mix_ratio - )[self.t_pad_tgt : -self.t_pad_tgt] - ) - s = t - pbar.update(1) - - audio_opt.append( - self.voice_conversion( - model, - net_g, - sid, - audio_pad[t:], - (pitch[:, t // self.window :] if t is not None else pitch) if pitch_guidance else None, - (pitchf[:, t // self.window :] if t is not None else pitchf) if pitch_guidance else None, - index, - big_tsr, - index_rate, - version, - protect, - (energy[:, t // self.window :] if t is not None else energy) if energy_use else None, - embedders_mix=embedders_mix, - embedders_mix_layers=embedders_mix_layers, - embedders_mix_ratio=embedders_mix_ratio - )[self.t_pad_tgt : -self.t_pad_tgt] - ) - - pbar.update(1) - - audio_opt = np.concatenate(audio_opt) - if rms_mix_rate != 1: - audio_opt = change_rms( - audio, - self.sample_rate, - audio_opt, - self.tgt_sr, - rms_mix_rate, - device=self.device - ).cpu().numpy() - - audio_max = np.abs(audio_opt).max() / 0.99 - if audio_max > 1: audio_opt /= audio_max - - if pitch_guidance: del pitch, pitchf - del sid - - clear_gpu_cache() - return audio_opt \ No newline at end of file diff --git a/main/inference/create_dataset.py b/main/inference/create_dataset.py deleted file mode 100644 index 5a15a1403933b47e91054537612e6a1e5db1f2d1..0000000000000000000000000000000000000000 --- a/main/inference/create_dataset.py +++ /dev/null @@ -1,449 +0,0 @@ -import os -import sys -import time -import torch -import yt_dlp -import shutil -import librosa -import argparse -import warnings - -import numpy as np -import soundfile as sf - -from urllib.parse import urlparse - -sys.path.append(os.getcwd()) - -from main.library.utils import strtobool -from main.inference.separate_music import _separate -from main.app.variables import config, logger, translations - -dataset_temp = "dataset_temp" - -def parse_arguments(): - parser = argparse.ArgumentParser() - parser.add_argument("--create_dataset", action='store_true') - parser.add_argument("--input_data", type=str, required=True) - parser.add_argument("--output_dirs", type=str, default="./dataset") - parser.add_argument("--skip_seconds", type=lambda x: bool(strtobool(x)), default=False) - parser.add_argument("--skip_start_audios", type=str, default="0") - parser.add_argument("--skip_end_audios", type=str, default="0") - parser.add_argument("--separate", type=lambda x: bool(strtobool(x)), default=True) - parser.add_argument("--model_name", type=str, default="MDXNET_Main") - parser.add_argument("--reverb_model", type=str, default="MDX-Reverb") - parser.add_argument("--denoise_model", type=str, default="Normal") - parser.add_argument("--sample_rate", type=int, default=48000) - parser.add_argument("--shifts", type=int, default=2) - parser.add_argument("--batch_size", type=int, default=1) - parser.add_argument("--overlap", type=float, default=0.25) - parser.add_argument("--aggression", type=int, default=5) - parser.add_argument("--hop_length", type=int, default=1024) - parser.add_argument("--window_size", type=int, default=512) - parser.add_argument("--segments_size", type=int, default=256) - parser.add_argument("--post_process_threshold", type=float, default=0.2) - parser.add_argument("--enable_tta", type=lambda x: bool(strtobool(x)), default=False) - parser.add_argument("--enable_denoise", type=lambda x: bool(strtobool(x)), default=False) - parser.add_argument("--high_end_process", type=lambda x: bool(strtobool(x)), default=False) - parser.add_argument("--enable_post_process", type=lambda x: bool(strtobool(x)), default=False) - parser.add_argument("--separate_reverb", type=lambda x: bool(strtobool(x)), default=False) - parser.add_argument("--clean_dataset", type=lambda x: bool(strtobool(x)), default=False) - parser.add_argument("--clean_strength", type=float, default=0.7) - - return parser.parse_args() - -def main(): - args = parse_arguments() - - ( - input_data, - output_dirs, - skip_seconds, - skip_start_audios, - skip_end_audios, - separate, - model_name, - reverb_model, - denoise_model, - sample_rate, - shifts, - batch_size, - overlap, - aggression, - hop_length, - window_size, - segments_size, - post_process_threshold, - enable_tta, - enable_denoise, - high_end_process, - enable_post_process, - separate_reverb, - clean_dataset, - clean_strength - ) = ( - args.input_data, - args.output_dirs, - args.skip_seconds, - args.skip_start_audios, - args.skip_end_audios, - args.separate, - args.model_name, - args.reverb_model, - args.denoise_model, - args.sample_rate, - args.shifts, - args.batch_size, - args.overlap, - args.aggression, - args.hop_length, - args.window_size, - args.segments_size, - args.post_process_threshold, - args.enable_tta, - args.enable_denoise, - args.high_end_process, - args.enable_post_process, - args.separate_reverb, - args.clean_dataset, - args.clean_strength - ) - - create_dataset( - input_data, - output_dirs, - skip_seconds, - skip_start_audios, - skip_end_audios, - separate, - model_name, - reverb_model, - denoise_model, - sample_rate, - shifts, - batch_size, - overlap, - aggression, - hop_length, - window_size, - segments_size, - post_process_threshold, - enable_tta, - enable_denoise, - high_end_process, - enable_post_process, - separate_reverb, - clean_dataset, - clean_strength - ) - -def create_dataset( - input_data, - output_dirs, - skip_seconds, - skip_start_audios, - skip_end_audios, - separate, - model_name, - reverb_model="MDX-Reverb", - denoise_model="Normal", - sample_rate=48000, - shifts=2, - batch_size=1, - overlap=0.25, - aggression=5, - hop_length=1024, - window_size=512, - segments_size=256, - post_process_threshold=0.2, - enable_tta=False, - enable_denoise=False, - high_end_process=False, - enable_post_process=False, - separate_reverb=False, - clean_dataset=False, - clean_strength=0.7 -): - log_data = { - translations["audio_path"]: input_data, - translations["output_path"]: output_dirs, - translations["skip"]: skip_seconds, - translations["separator_tab"]: separate, - translations["modelname"]: model_name, - translations["dereveb_audio"]: separate_reverb, - translations["sr"]: sample_rate, - translations["shift"]: shifts, - translations["batch_size"]: batch_size, - translations["overlap"]: overlap, - translations["aggression"]: aggression, - translations["hop_length"]: hop_length, - translations["window_size"]: window_size, - translations["segments_size"]: segments_size, - translations["post_process_threshold"]: post_process_threshold, - translations["enable_tta"]: enable_tta, - translations["denoise_mdx"]: enable_denoise, - translations["high_end_process"]: high_end_process, - translations["enable_post_process"]: enable_post_process, - translations["clear_dataset"]: clean_dataset, - translations["clean_strength"]: clean_strength, - translations["dereveb_model"]: reverb_model, - translations["denoise_model"]: denoise_model, - translations["skip_start"]: skip_start_audios, - translations["skip_end"]: skip_end_audios - } - - for key, value in log_data.items(): - logger.debug(f"{key}: {value}") - - start_time = time.time() - inputs_data = input_data.replace(", ", ",").split(",") - - pid_path = os.path.join("assets", "create_dataset_pid.txt") - with open(pid_path, "w") as pid_file: - pid_file.write(str(os.getpid())) - - try: - if os.path.exists(dataset_temp): shutil.rmtree(dataset_temp, ignore_errors=True) - else: os.makedirs(dataset_temp, exist_ok=True) - - audio_path = [ - downloader( - url, - f"audio_{str(inputs_data.index(url))}" - ) if is_url(url) else url - for url in inputs_data - ] - - if skip_seconds: - skip_start_audios, skip_end_audios = ( - skip_start_audios.replace(", ", ",").split(","), - skip_end_audios.replace(", ", ",").split(",") - ) - - if len(skip_start_audios) < len(audio_path) or len(skip_end_audios) < len(audio_path): - logger.warning(translations["skip len(audio_path) or len(skip_end_audios) > len(audio_path): - logger.warning(translations["skip>audio"]) - sys.exit(1) - else: - audio_path = [ - skip_duration( - audio, - skip_start_audio, - skip_end_audio - ) - for audio, skip_start_audio, skip_end_audio in zip( - audio_path, - skip_start_audios, - skip_end_audios - ) - ] - - if separate: - audio_path = [ - separate_main( - audio, - audio_path.index(audio), - model_name, - sample_rate, - reverb_model, - denoise_model, - shifts, - batch_size, - overlap, - aggression, - hop_length, - window_size, - segments_size, - post_process_threshold, - enable_tta, - enable_denoise, - high_end_process, - enable_post_process, - separate_reverb - ) - for audio in audio_path - ] - - if clean_dataset: - from main.library.audio.noisereduce import TorchGate - - tg = TorchGate( - sr, - prop_decrease=clean_strength - ).to(config.device) - - for audio in audio_path: - data, sr = read_file(audio) - if len(data.shape) > 1: - data = librosa.to_mono(data.T) - - if sr != sample_rate: - data = librosa.resample( - data, - orig_sr=sr, - target_sr=sample_rate, - res_type="soxr_vhq" - ) - - if clean_dataset: - data = tg( - torch.from_numpy(data).unsqueeze(0).to(config.device).float() - ).squeeze(0).cpu().detach().numpy() - - sf.write(audio, data, sr) - output_path = os.path.join(output_dirs, os.path.basename(audio)) - - if os.path.exists(output_path): os.remove(output_path) - shutil.move(audio, output_path) - - if os.path.exists(dataset_temp): shutil.rmtree(dataset_temp, ignore_errors=True) - except Exception as e: - logger.error(f"{translations['create_dataset_error']}: {e}") - import traceback - logger.error(traceback.format_exc()) - - elapsed_time = time.time() - start_time - if os.path.exists(pid_path): os.remove(pid_path) - - logger.info(translations["create_dataset_success"].format(elapsed_time=f"{elapsed_time:.2f}")) - return output_dirs - -def separate_main( - input_path, - index, - model_name, - sample_rate, - reverb_model="MDX-Reverb", - denoise_model="Normal", - shifts=2, - batch_size=1, - overlap=0.25, - aggression=5, - hop_length=1024, - window_size=512, - segments_size=256, - post_process_threshold=0.2, - enable_tta=False, - enable_denoise=False, - high_end_process=False, - enable_post_process=False, - separate_reverb=False -): - original_vocals, _, _, _ = _separate( - input_path, - dataset_temp, - model_name, - reverb_model=reverb_model, - denoise_model=denoise_model, - sample_rate=sample_rate, - shifts=shifts, - batch_size=batch_size, - overlap=overlap, - aggression=aggression, - hop_length=hop_length, - window_size=window_size, - segments_size=segments_size, - post_process_threshold=post_process_threshold, - enable_tta=enable_tta, - enable_denoise=enable_denoise, - high_end_process=high_end_process, - enable_post_process=enable_post_process, - separate_reverb=separate_reverb - ) - - vocals = os.path.join(dataset_temp, f"dataset_{index}.wav") - os.rename(original_vocals, vocals) - - return vocals - -def is_url(path): - try: - result = urlparse(path) - return all([result.scheme, result.netloc]) - except ValueError: - return False - -def downloader( - url, - name -): - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - - ydl_opts = { - "format": "bestaudio/best", - "outtmpl": os.path.join(dataset_temp, f"{name}"), - "postprocessors": [{ - "key": "FFmpegExtractAudio", - "preferredcodec": "wav", - "preferredquality": "192" - }], - "no_warnings": True, - "noplaylist": True, - "noplaylist": True, - "verbose": False - } - - logger.info(f"{translations['starting_download']}: {url}...") - - with yt_dlp.YoutubeDL(ydl_opts) as ydl: - ydl.extract_info(url) - logger.info(f"{translations['download_success']}: {url}") - - return os.path.join(dataset_temp, f"{name}" + ".wav") - -def read_file(file): - try: - data, sr = sf.read(file, dtype=np.float32) - except: - data, sr = librosa.load(file, sr=None) - - return data, sr - -def skip_duration( - audio, - skip_start_audio, - skip_end_audio -): - skip_start(audio, int(skip_start_audio)) - skip_end(audio, int(skip_end_audio)) - - return audio - -def skip_start( - input_file, - seconds -): - data, sr = read_file(input_file) - total_duration = len(data) / sr - - if seconds <= 0: - logger.warning(translations["=<0"]) - elif seconds >= total_duration: - logger.warning(translations["skip_warning"].format(seconds=seconds, total_duration=f"{total_duration:.2f}")) - else: - logger.info(f"{translations['skip_start']}: {input_file}...") - sf.write(input_file, data[int(seconds * sr):], sr) - - logger.info(translations["skip_start_audio"].format(input_file=input_file)) - -def skip_end( - input_file, - seconds -): - data, sr = read_file(input_file) - total_duration = len(data) / sr - - if seconds <= 0: - logger.warning(translations["=<0"]) - elif seconds > total_duration: - logger.warning(translations["skip_warning"].format(seconds=seconds, total_duration=f"{total_duration:.2f}")) - else: - logger.info(f"{translations['skip_end']}: {input_file}...") - sf.write(input_file, data[:-int(seconds * sr)], sr) - - logger.info(translations["skip_end_audio"].format(input_file=input_file)) - -if __name__ == "__main__": main() \ No newline at end of file diff --git a/main/inference/create_index.py b/main/inference/create_index.py deleted file mode 100644 index a15330e45cddab654c3401871f65d79c38517395..0000000000000000000000000000000000000000 --- a/main/inference/create_index.py +++ /dev/null @@ -1,122 +0,0 @@ -import os -import sys -import faiss -import argparse - -import numpy as np - -from multiprocessing import cpu_count -from sklearn.cluster import MiniBatchKMeans - -sys.path.append(os.getcwd()) - -from main.app.variables import logger, translations, configs - -def parse_arguments(): - parser = argparse.ArgumentParser() - parser.add_argument("--create_index", action='store_true') - parser.add_argument("--model_name", type=str, required=True) - parser.add_argument("--rvc_version", type=str, default="v2") - parser.add_argument("--index_algorithm", type=str, default="Auto") - parser.add_argument("--nprobe", type=int, default=1) - - return parser.parse_args() - -def main(): - args = parse_arguments() - - exp_dir = os.path.join(configs["logs_path"], args.model_name) - version, index_algorithm, nprobe = args.rvc_version, args.index_algorithm, args.nprobe - - log_data = { - translations['modelname']: args.model_name, - translations['model_path']: exp_dir, - translations['training_version']: version, - translations['index_algorithm_info']: index_algorithm, - translations['nprobe']: nprobe - } - - for key, value in log_data.items(): - logger.debug(f"{key}: {value}") - - try: - npys = [] - feature_dir = os.path.join(exp_dir, f"{version}_extracted") - model_name = os.path.basename(exp_dir) - - for name in sorted(os.listdir(feature_dir)): - npys.append(np.load(os.path.join(feature_dir, name))) - - big_npy = np.concatenate(npys, axis=0) - big_npy_idx = np.arange(big_npy.shape[0]) - - np.random.shuffle(big_npy_idx) - big_npy = big_npy[big_npy_idx] - - if big_npy.shape[0] > 2e5 and (index_algorithm == "Auto" or index_algorithm == "KMeans"): - big_npy = ( - MiniBatchKMeans( - n_clusters=10000, - verbose=True, - batch_size=256 * cpu_count(), - compute_labels=False, - init="random" - ).fit(big_npy).cluster_centers_ - ) - - np.save( - os.path.join(exp_dir, "total_fea.npy"), - big_npy - ) - - n_ivf = min(int(16 * np.sqrt(big_npy.shape[0])), big_npy.shape[0] // 39) - - index_trained = faiss.index_factory( - 256 if version == "v1" else 768, - f"IVF{n_ivf},Flat" - ) - - index_ivf_trained = faiss.extract_index_ivf(index_trained) - index_ivf_trained.nprobe = nprobe - index_trained.train(big_npy) - - faiss.write_index( - index_trained, - os.path.join( - exp_dir, - f"trained_IVF{n_ivf}_Flat_nprobe_{index_ivf_trained.nprobe}_{model_name}_{version}.index" - ) - ) - - index_added = faiss.index_factory( - 256 if version == "v1" else 768, - f"IVF{n_ivf},Flat" - ) - - index_ivf_added = faiss.extract_index_ivf(index_added) - index_ivf_added.nprobe = nprobe - index_added.train(big_npy) - - batch_size_add = 8192 - for i in range(0, big_npy.shape[0], batch_size_add): - index_added.add( - big_npy[i : i + batch_size_add] - ) - - index_filepath_added = os.path.join( - exp_dir, - f"added_IVF{n_ivf}_Flat_nprobe_{index_ivf_added.nprobe}_{model_name}_{version}.index" - ) - - faiss.write_index( - index_added, - index_filepath_added - ) - - logger.info(f"{translations['save_index']} '{index_filepath_added}'") - except Exception as e: - logger.error(f"{translations['create_index_error']}: {e}") - import traceback - logger.debug(traceback.format_exc()) - -if __name__ == "__main__": main() \ No newline at end of file diff --git a/main/inference/create_reference.py b/main/inference/create_reference.py deleted file mode 100644 index 97f3983c012ec18321c067c4c94e92b673d3f430..0000000000000000000000000000000000000000 --- a/main/inference/create_reference.py +++ /dev/null @@ -1,292 +0,0 @@ -import os -import sys -import time -import torch -import shutil -import warnings -import argparse - -import numpy as np - -from tqdm import tqdm - -sys.path.append(os.getcwd()) - -from main.library.audio.audio import load_audio -from main.app.variables import config, logger, translations, configs -from main.library.utils import load_embedders_model, extract_features, check_assets, strtobool - -warnings.filterwarnings("ignore") - -F0_MIN, F0_MAX, HOP_SIZE, SAMPLE_RATE, FRAME_LENGTH = configs.get("f0_min", 50), configs.get("f0_max", 1100), 160, 16000, 2048 - -def parse_arguments(): - parser = argparse.ArgumentParser() - parser.add_argument("--create_reference", action='store_true') - parser.add_argument("--audio_path", type=str, required=True) - parser.add_argument("--reference_name", type=str, default="reference") - parser.add_argument("--pitch_guidance", type=lambda x: bool(strtobool(x)), default=True) - parser.add_argument("--use_energy", type=lambda x: bool(strtobool(x)), default=False) - parser.add_argument("--version", type=str, default="v2") - parser.add_argument("--embedder_model", type=str, default="hubert_base") - parser.add_argument("--embedders_mode", type=str, default="fairseq") - parser.add_argument("--f0_method", type=str, default="rmvpe") - parser.add_argument("--predictor_onnx", type=lambda x: bool(strtobool(x)), default=False) - parser.add_argument("--f0_up_key", type=int, default=0) - parser.add_argument("--filter_radius", type=int, default=3) - parser.add_argument("--f0_autotune", type=lambda x: bool(strtobool(x)), default=False) - parser.add_argument("--f0_autotune_strength", type=float, default=1) - parser.add_argument("--proposal_pitch", type=lambda x: bool(strtobool(x)), default=False) - parser.add_argument("--proposal_pitch_threshold", type=float, default=255.0) - parser.add_argument("--alpha", type=float, default=0.5) - parser.add_argument("--embedders_mix", type=lambda x: bool(strtobool(x)), default=False) - parser.add_argument("--embedders_mix_layers", type=int, default=9, required=False) - parser.add_argument("--embedders_mix_ratio", type=float, default=0.5) - - return parser.parse_args() - -def main(): - args = parse_arguments() - - ( - audio_path, - reference_name, - pitch_guidance, - use_energy, - version, - embedder_model, - embedders_mode, - f0_method, - predictor_onnx, - f0_up_key, - filter_radius, - f0_autotune, - f0_autotune_strength, - proposal_pitch, - proposal_pitch_threshold, - alpha, - embedders_mix, - embedders_mix_layers, - embedders_mix_ratio - ) = ( - args.audio_path, - args.reference_name, - args.pitch_guidance, - args.use_energy, - args.version, - args.embedder_model, - args.embedders_mode, - args.f0_method, - args.predictor_onnx, - args.f0_up_key, - args.filter_radius, - args.f0_autotune, - args.f0_autotune_strength, - args.proposal_pitch, - args.proposal_pitch_threshold, - args.alpha, - args.embedders_mix, - args.embedders_mix_layers, - args.embedders_mix_ratio - ) - - log_data = { - translations["audio_path"]: audio_path, - translations["reference_name"]: reference_name, - translations['extract_f0']: pitch_guidance, - translations["train&energy"]: use_energy, - translations['training_version']: version, - translations['hubert_model']: embedder_model, - translations["embed_mode"]: embedders_mode, - translations['f0_method']: f0_method, - translations["predictor_onnx"]: predictor_onnx, - translations["pitch"]: f0_up_key, - translations["filter_radius"]: filter_radius, - translations["autotune"]: f0_autotune, - translations["autotune_rate_info"]: f0_autotune_strength, - translations["proposal_pitch"]: proposal_pitch, - translations["proposal_pitch_threshold"]: proposal_pitch_threshold, - translations["alpha_label"]: alpha, - translations["embedders_mix"]: embedders_mix, - translations["embedders_mix_layers"]: embedders_mix_layers, - translations["embedders_mix_ratio"]: embedders_mix_ratio, - } - - for key, value in log_data.items(): - logger.debug(f"{key}: {value}") - - check_assets( - f0_method, - embedder_model, - predictor_onnx, - embedders_mode - ) - - create_reference( - audio_path, - reference_name, - pitch_guidance, - use_energy, - version, - embedder_model, - embedders_mode, - f0_method, - predictor_onnx, - f0_up_key, - filter_radius, - f0_autotune, - f0_autotune_strength, - proposal_pitch, - proposal_pitch_threshold, - alpha, - embedders_mix, - embedders_mix_layers, - embedders_mix_ratio - ) - -def create_reference( - audio_path, - reference_name, - pitch_guidance = True, - use_energy = False, - version = "v2", - embedder_model = "hubert_base", - embedders_mode = "fairseq", - f0_method = "rmvpe", - predictor_onnx = False, - f0_up_key = 0, - filter_radius = 3, - f0_autotune = False, - f0_autotune_strength = 1, - proposal_pitch = False, - proposal_pitch_threshold = 255.0, - alpha = 0.5, - embedders_mix = False, - embedders_mix_layers = 9, - embedders_mix_ratio = 0.5 -): - device = config.device - is_half = config.is_half - - if not audio_path: - logger.warning(translations["not_found_audio"]) - sys.exit(1) - - output_reference = os.path.join( - configs["reference_path"], - f"{reference_name}_{version}_{embedder_model}_{pitch_guidance}_{use_energy}" - ) - - if os.path.exists(output_reference): shutil.rmtree(reference_name, ignore_errors=True) - - os.makedirs(output_reference, exist_ok=True) - logger.info(translations["start_create_reference"]) - start_time = time.time() - - with tqdm(total=5, desc=translations["create_reference"], ncols=100, unit="a") as pbar: - audio = load_audio(audio_path, sample_rate=SAMPLE_RATE) - pbar.update(1) - - audio_max = np.abs(audio).max() / 0.95 - if audio_max > 1: audio /= audio_max - - trimmed_len = (len(audio) // 320) * 320 - audio = audio[:trimmed_len] - - audio_pad = torch.nn.functional.pad( - torch.from_numpy(audio).to( - torch.float16 if is_half else torch.float32 - ).to(device).unsqueeze(0), - (40, 40), - mode="reflect" - ) - pbar.update(1) - - embedder = load_embedders_model(embedder_model, embedders_mode) - if isinstance(embedder, torch.nn.Module): - embedder = embedder.to(torch.float16 if is_half else torch.float32).eval().to(device) - - with torch.no_grad(): - feats = extract_features( - embedder, - audio_pad.view(1, -1), - version, - mix=embedders_mix, - mix_layers=embedders_mix_layers, - mix_ratio=embedders_mix_ratio - ) - - np.save( - os.path.join(output_reference, "feats.npy"), - feats.squeeze(0).float().cpu().numpy(), - allow_pickle=False - ) - - pbar.update(1) - - if pitch_guidance: - from main.library.predictors.Generator import Generator - - generator = Generator( - sample_rate=SAMPLE_RATE, - hop_length=HOP_SIZE, - f0_min=F0_MIN, - f0_max=F0_MAX, - alpha=alpha, - is_half=is_half, - device=device, - predictor_onnx=predictor_onnx - ) - - pitch, pitchf = generator.calculator( - x_pad=config.x_pad, - f0_method=f0_method, - x=audio, - f0_up_key=f0_up_key, - p_len=audio.shape[0] // 160 + 1, - filter_radius=filter_radius, - f0_autotune=f0_autotune, - f0_autotune_strength=f0_autotune_strength, - manual_f0=None, - proposal_pitch=proposal_pitch, - proposal_pitch_threshold=proposal_pitch_threshold - ) - - np.save( - os.path.join(output_reference, "pitch_coarse.npy"), - pitch, - allow_pickle=False - ) - - np.save( - os.path.join(output_reference, "pitch_fine.npy"), - pitchf, - allow_pickle=False - ) - - pbar.update(1) - - if use_energy: - from main.inference.extracting.rms import RMSEnergyExtractor - - rms = RMSEnergyExtractor( - frame_length=FRAME_LENGTH, - hop_length=HOP_SIZE, - center=True, - pad_mode="reflect" - ).to(device).eval() - - with torch.no_grad(): - energy = rms(audio_pad) - - np.save( - os.path.join(output_reference, "energy.npy"), - energy.float().cpu().numpy(), - allow_pickle=False - ) - - pbar.update(1) - - logger.info(translations["create_reference_success"].format(elapsed_time=f"{(time.time() - start_time):.2f}")) - -if __name__ == "__main__": main() \ No newline at end of file diff --git a/main/inference/extracting/embedding.py b/main/inference/extracting/embedding.py deleted file mode 100644 index d1cf8632fe079853aae35cfb1a5243973521dbd9..0000000000000000000000000000000000000000 --- a/main/inference/extracting/embedding.py +++ /dev/null @@ -1,147 +0,0 @@ -import os -import gc -import sys -import time -import tqdm -import torch -import traceback -import concurrent.futures - -import numpy as np - -sys.path.append(os.getcwd()) - -from main.library.audio.audio import load_audio -from main.app.variables import logger, translations, config -from main.inference.extracting.setup_path import setup_paths -from main.library.utils import load_embedders_model, extract_features - -def process_file_embedding( - files, - embedder_model, - embedders_mode, - device, - version, - is_half, - threads, - embedders_mix = False, - embedders_mix_layers = 9, - embedders_mix_ratio = 0.5 -): - dtype = torch.float16 if is_half else torch.float32 - - model = load_embedders_model( - embedder_model, - embedders_mode - ) - - if isinstance(model, torch.nn.Module): model = model.to(device).to(dtype).eval() - - def worker(file_info): - try: - file, out_path = file_info - - out_file_path = os.path.join( - out_path, - os.path.basename(file.replace("wav", "npy")) - ) if os.path.isdir(out_path) else out_path - - if os.path.exists(out_file_path): return - - feats = torch.from_numpy( - load_audio(file, sample_rate=16000) - ).to(device).to(dtype) - - with torch.no_grad(): - feats = extract_features( - model, - feats.view(1, -1), - version, - mix=embedders_mix, - mix_layers=embedders_mix_layers, - mix_ratio=embedders_mix_ratio - ) - - feats = feats.squeeze(0).float().cpu().numpy() - - if not np.isnan(feats).any(): - np.save( - out_file_path, - feats, - allow_pickle=False - ) - else: logger.warning(f"{file} {translations['NaN']}") - except: - logger.debug(traceback.format_exc()) - - with tqdm.tqdm(total=len(files), ncols=100, unit="p", leave=True) as pbar: - with concurrent.futures.ThreadPoolExecutor(max_workers=threads) as executor: - for _ in concurrent.futures.as_completed([executor.submit(worker, f) for f in files]): - pbar.update(1) - -def run_embedding_extraction( - exp_dir, - version, - num_processes, - devices, - embedder_model, - embedders_mode, - is_half, - embedders_mix = False, - embedders_mix_layers = 9, - embedders_mix_ratio = 0.5 -): - wav_path, out_path = setup_paths(exp_dir, version) - logger.info(translations["start_extract_hubert"]) - - if ( - (config.device.startswith("ocl") and embedders_mode == "onnx") or - config.device.startswith("privateuseone") - ): - num_processes = 1 - - paths = sorted([ - (os.path.join(wav_path, file), out_path) - for file in os.listdir(wav_path) if file.endswith(".wav") - ]) - - start_time = time.time() - with concurrent.futures.ProcessPoolExecutor(max_workers=len(devices)) as executor: - concurrent.futures.wait([ - executor.submit( - process_file_embedding, - paths[i::len(devices)], - embedder_model, embedders_mode, - devices[i], - version, - is_half, - num_processes // len(devices), - embedders_mix, - embedders_mix_layers, - embedders_mix_ratio - ) - for i in range(len(devices)) - ]) - - gc.collect() - logger.info(translations["extract_hubert_success"].format(elapsed_time=f"{(time.time() - start_time):.2f}")) - -def create_mute_file(version, embedder_model, embedders_mode, is_half): - start_time = time.time() - logger.info(translations["start_extract_hubert"]) - - process_file_embedding( - [( - os.path.join(config.configs["logs_path"], "mute", "sliced_audios_16k", "mute.wav"), - os.path.join(config.configs["logs_path"], "mute", f"{version}_extracted", f"mute_{embedder_model}.npy") - )], - embedder_model, - embedders_mode, - config.device, - version, - is_half, - 1 - ) - - gc.collect() - logger.info(translations["extract_hubert_success"].format(elapsed_time=f"{(time.time() - start_time):.2f}")) \ No newline at end of file diff --git a/main/inference/extracting/extract.py b/main/inference/extracting/extract.py deleted file mode 100644 index f48fb7575d43aa980fafbc9c12b12aa648d5c732..0000000000000000000000000000000000000000 --- a/main/inference/extracting/extract.py +++ /dev/null @@ -1,190 +0,0 @@ -import os -import sys -import logging -import argparse -import warnings - -import torch.multiprocessing as mp - -sys.path.append(os.getcwd()) - -from main.library.utils import check_assets, strtobool -from main.inference.extracting.rms import run_rms_extraction -from main.inference.extracting.feature import run_pitch_extraction -from main.app.variables import config, logger, translations, configs -from main.inference.extracting.embedding import run_embedding_extraction -from main.inference.extracting.preparing_files import generate_config, generate_filelist - -if not config.debug_mode: - warnings.filterwarnings("ignore") - for l in ["torch", "faiss", "httpx", "httpcore", "faiss.loader", "numba.core", "urllib3", "matplotlib"]: - logging.getLogger(l).setLevel(logging.ERROR) - -def parse_arguments(): - parser = argparse.ArgumentParser() - parser.add_argument("--extract", action='store_true') - parser.add_argument("--model_name", type=str, required=True) - parser.add_argument("--rvc_version", type=str, default="v2") - parser.add_argument("--f0_method", type=str, default="rmvpe") - parser.add_argument("--pitch_guidance", type=lambda x: bool(strtobool(x)), default=True) - parser.add_argument("--hop_length", type=int, default=128) - parser.add_argument("--cpu_cores", type=int, default=2) - parser.add_argument("--gpu", type=str, default="-") - parser.add_argument("--sample_rate", type=int, required=True) - parser.add_argument("--embedder_model", type=str, default="hubert_base") - parser.add_argument("--predictor_onnx", type=lambda x: bool(strtobool(x)), default=False) - parser.add_argument("--embedders_mode", type=str, default="fairseq") - parser.add_argument("--f0_autotune", type=lambda x: bool(strtobool(x)), default=False) - parser.add_argument("--f0_autotune_strength", type=float, default=1) - parser.add_argument("--rms_extract", type=lambda x: bool(strtobool(x)), default=False) - parser.add_argument("--alpha", type=float, default=0.5) - parser.add_argument("--include_mutes", type=int, default=2) - parser.add_argument("--embedders_mix", type=lambda x: bool(strtobool(x)), default=False) - parser.add_argument("--embedders_mix_layers", type=int, default=9, required=False) - parser.add_argument("--embedders_mix_ratio", type=float, default=0.5) - parser.add_argument("--architecture", type=str, default="RVC") - - return parser.parse_args() - -def main(): - args = parse_arguments() - - ( - f0_method, - hop_length, - num_processes, - gpus, version, - pitch_guidance, - sample_rate, - embedder_model, - predictor_onnx, - embedders_mode, - f0_autotune, - f0_autotune_strength, - rms_extract, - alpha, - include_mutes, - embedders_mix, - embedders_mix_layers, - embedders_mix_ratio, - architecture - ) = ( - args.f0_method, - args.hop_length, - args.cpu_cores, - args.gpu, - args.rvc_version, - args.pitch_guidance, - args.sample_rate, - args.embedder_model, - args.predictor_onnx, - args.embedders_mode, - args.f0_autotune, - args.f0_autotune_strength, - args.rms_extract, - args.alpha, - args.include_mutes, - args.embedders_mix, - args.embedders_mix_layers, - args.embedders_mix_ratio, - args.architecture - ) - - check_assets(f0_method, embedder_model, predictor_onnx=predictor_onnx, embedders_mode=embedders_mode) - exp_dir = os.path.join(configs["logs_path"], args.model_name) - num_processes = max(1, num_processes) - - devices = ["cpu"] if gpus == "-" else [ - ( - f"cuda:{idx}" - ) if config.device.startswith("cuda") else ( - f"xpu:{idx}" if config.device.startswith("xpu") else f"{'ocl' if config.device.startswith('ocl') else 'privateuseone'}:{idx}" - ) - for idx in gpus.split("-") - ] - - log_data = { - translations['modelname']: args.model_name, - translations['export_process']: exp_dir, - translations['f0_method']: f0_method, - translations['pretrain_sr']: sample_rate, - translations['cpu_core']: num_processes, - "Gpu": gpus, - translations['hop_length']: hop_length, - translations['training_version']: version, - translations['extract_f0']: pitch_guidance, - translations['hubert_model']: embedder_model, - translations["predictor_onnx"]: predictor_onnx, - translations["embed_mode"]: embedders_mode, - translations["train&energy"]: rms_extract, - translations["alpha_label"]: alpha, - translations["include_mutes"]: include_mutes, - translations["embedders_mix"]: embedders_mix, - translations["embedders_mix_layers"]: embedders_mix_layers, - translations["embedders_mix_ratio"]: embedders_mix_ratio, - translations["architecture"]: architecture - } - - for key, value in log_data.items(): - logger.debug(f"{key}: {value}") - - pid_path = os.path.join(exp_dir, "extract_pid.txt") - with open(pid_path, "w") as pid_file: - pid_file.write(str(os.getpid())) - - try: - run_pitch_extraction( - exp_dir, - f0_method, - hop_length, - num_processes, - devices, - predictor_onnx, - config.is_half, - f0_autotune, - f0_autotune_strength, - alpha - ) - run_embedding_extraction( - exp_dir, - version, - num_processes, - devices, - embedder_model, - embedders_mode, - config.is_half, - embedders_mix, - embedders_mix_layers, - embedders_mix_ratio - ) - run_rms_extraction( - exp_dir, - num_processes, - devices, - rms_extract - ) - generate_config( - version, - sample_rate, - exp_dir, - architecture - ) - generate_filelist( - pitch_guidance, - exp_dir, - version, - sample_rate, - embedders_mode, - embedder_model, - rms_extract, - include_mutes - ) - except Exception as e: - logger.error(f"{translations['extract_error']}: {e}") - - if os.path.exists(pid_path): os.remove(pid_path) - logger.info(f"{translations['extract_success']} {args.model_name}.") - -if __name__ == "__main__": - mp.set_start_method("spawn", force=True) - main() \ No newline at end of file diff --git a/main/inference/extracting/feature.py b/main/inference/extracting/feature.py deleted file mode 100644 index 76b5c39cf30c48edb1f20c82271da19eb1998a72..0000000000000000000000000000000000000000 --- a/main/inference/extracting/feature.py +++ /dev/null @@ -1,177 +0,0 @@ -import os -import gc -import sys -import tqdm -import time -import traceback -import concurrent.futures - -import numpy as np - -sys.path.append(os.getcwd()) - -from main.library.audio.audio import load_audio -from main.inference.extracting.setup_path import setup_paths -from main.app.variables import config, logger, translations, configs - -class FeatureInput: - def __init__( - self, - is_half=config.is_half, - device=config.device - ): - self.sample_rate = 16000 - self.f0_min = configs.get("f0_min", 50) - self.f0_max = configs.get("f0_max", 1100) - self.device = device - self.is_half = is_half - - def process_file( - self, - file_info, - f0_method, - hop_length, - predictor_onnx, - f0_autotune, - f0_autotune_strength, - alpha - ): - if not hasattr(self, "f0_gen"): - from main.library.predictors.Generator import Generator - - self.f0_gen = Generator( - self.sample_rate, - hop_length, - self.f0_min, - self.f0_max, - alpha, - self.is_half, - self.device, - predictor_onnx - ) - - inp_path, opt_path1, opt_path2, file_inp = file_info - if os.path.exists(opt_path1 + ".npy") and os.path.exists(opt_path2 + ".npy"): return - - try: - pitch, pitchf = self.f0_gen.calculator( - x_pad=config.x_pad, - f0_method=f0_method, - x=load_audio(file_inp, sample_rate=self.sample_rate), - f0_up_key=0, - p_len=None, - filter_radius=3, - f0_autotune=f0_autotune, - f0_autotune_strength=f0_autotune_strength, - manual_f0=None, - proposal_pitch=False, - proposal_pitch_threshold=0.0 - ) - - np.save( - opt_path2, - pitchf, - allow_pickle=False - ) - - np.save( - opt_path1, - pitch, - allow_pickle=False - ) - except Exception as e: - logger.info(f"{translations['extract_file_error']} {inp_path}: {e}") - logger.debug(traceback.format_exc()) - - def process_files( - self, - files, - f0_method, - hop_length, - predictor_onnx, - device, - is_half, - threads, - f0_autotune, - f0_autotune_strength, - alpha - ): - self.device = device - self.is_half = is_half - - def worker(file_info): - self.process_file( - file_info, - f0_method, - hop_length, - predictor_onnx, - f0_autotune, - f0_autotune_strength, - alpha - ) - - with tqdm.tqdm(total=len(files), ncols=100, unit="p", leave=True) as pbar: - with concurrent.futures.ThreadPoolExecutor(max_workers=threads) as executor: - for _ in concurrent.futures.as_completed([executor.submit(worker, f) for f in files]): - pbar.update(1) - -def run_pitch_extraction( - exp_dir, - f0_method, - hop_length, - num_processes, - devices, - predictor_onnx, - is_half, - f0_autotune, - f0_autotune_strength, - alpha -): - input_root, *output_roots = setup_paths(exp_dir) - output_root1, output_root2 = output_roots if len(output_roots) == 2 else (output_roots[0], None) - - logger.info(translations["extract_f0_method"].format(num_processes=num_processes, f0_method=f0_method)) - - if config.device.startswith(("ocl", "privateuseone")) and ( - "crepe" in f0_method or - "fcpe" in f0_method or - "rmvpe" in f0_method or - "penn" in f0_method or - "swift" in f0_method - ): - num_processes = 1 - - paths = [ - ( - os.path.join(input_root, name), - os.path.join(output_root1, name) if output_root1 else None, - os.path.join(output_root2, name) if output_root2 else None, - os.path.join(input_root, name) - ) - for name in sorted(os.listdir(input_root)) - if "spec" not in name - ] - - start_time = time.time() - feature_input = FeatureInput() - - with concurrent.futures.ProcessPoolExecutor(max_workers=len(devices)) as executor: - concurrent.futures.wait([ - executor.submit( - feature_input.process_files, - paths[i::len(devices)], - f0_method, - hop_length, - predictor_onnx, - devices[i], - is_half, - num_processes // len(devices), - f0_autotune, - f0_autotune_strength, - alpha - ) - for i in range(len(devices)) - ]) - - gc.collect() - logger.info(translations["extract_f0_success"].format(elapsed_time=f"{(time.time() - start_time):.2f}")) \ No newline at end of file diff --git a/main/inference/extracting/preparing_files.py b/main/inference/extracting/preparing_files.py deleted file mode 100644 index 072bac18759e95f73f6a5d02714b2ca804577433..0000000000000000000000000000000000000000 --- a/main/inference/extracting/preparing_files.py +++ /dev/null @@ -1,165 +0,0 @@ -import os -import sys -import json -import shutil - -from random import shuffle - -sys.path.append(os.getcwd()) - -from main.app.core.ui import configs, config -from main.inference.extracting.embedding import create_mute_file - -def mute_file( - embedders_mode, - embedders_model, - mute_base_path, - rvc_version -): - if embedders_mode.startswith("whisper"): - mute_file = f"mute_{embedders_model}.npy" - else: - mute_file = { - "contentvec_base": "mute.npy", - "hubert_base": "mute.npy", - "vietnamese_hubert_base": "mute_vietnamese.npy", - "japanese_hubert_base": "mute_japanese.npy", - "korean_hubert_base": "mute_korean.npy", - "chinese_hubert_base": "mute_chinese.npy", - "portuguese_hubert_base": "mute_portuguese.npy", - "spin-v1": "mute_spin-v1.npy", - "spin-v2": "mute_spin-v2.npy" - }.get(embedders_model, None) - - if mute_file is None or not os.path.exists(os.path.join(mute_base_path, f"{rvc_version}_extracted", mute_file)): - create_mute_file( - rvc_version, - embedders_model, - embedders_mode, - config.is_half - ) - mute_file = f"mute_{embedders_model}.npy" - - return os.path.join(mute_base_path, f"{rvc_version}_extracted", mute_file) - -def generate_config(rvc_version, sample_rate, model_path, architecture = "RVC"): - config_save_path = os.path.join(model_path, "config.json") - - if not os.path.exists(config_save_path): - shutil.copy( - os.path.join("main", "configs", rvc_version, f"{sample_rate}.json"), - config_save_path - ) - - if os.path.exists(config_save_path) and architecture == "SVC": - with open(config_save_path, "r", encoding="utf-8") as f: - data = json.load(f) - data["data"]["hop_length"] = 441 - - with open(config_save_path, "w", encoding="utf-8") as f: - json.dump(data, f, indent=4) - -def generate_filelist( - pitch_guidance, - model_path, - rvc_version, - sample_rate, - embedders_mode = "fairseq", - embedder_model = "hubert_base", - rms_extract = False, - include_mutes = 2 -): - f0_dir, f0nsf_dir, energy_dir = None, None, None - - gt_wavs_dir, feature_dir = ( - os.path.join(model_path, "sliced_audios"), - os.path.join(model_path, f"{rvc_version}_extracted") - ) - - if pitch_guidance: - f0_dir, f0nsf_dir = ( - os.path.join(model_path, "f0"), - os.path.join(model_path, "f0_voiced") - ) - - if rms_extract: - energy_dir = os.path.join(model_path, "energy") - - gt_wavs_files, feature_files = ( - set(name.split(".")[0] for name in os.listdir(gt_wavs_dir)), - set(name.split(".")[0] for name in os.listdir(feature_dir)) - ) - - names = gt_wavs_files & feature_files - - if pitch_guidance: - names = ( - names & - set(name.split(".")[0] for name in os.listdir(f0_dir)) & - set(name.split(".")[0] for name in os.listdir(f0nsf_dir)) - ) - - if rms_extract: - names = ( - names & - set(name.split(".")[0] for name in os.listdir(energy_dir)) - ) - - options, sids = [], [] - mute_base_path = os.path.join(configs["logs_path"], "mute") - - for name in names: - sid = name.split("_")[0] - if sid not in sids: sids.append(sid) - - option = { - True: { - True: f"{gt_wavs_dir}/{name}.wav|{feature_dir}/{name}.npy|{f0_dir}/{name}.wav.npy|{f0nsf_dir}/{name}.wav.npy|{energy_dir}/{name}.wav.npy|{sid}", - False: f"{gt_wavs_dir}/{name}.wav|{feature_dir}/{name}.npy|{f0_dir}/{name}.wav.npy|{f0nsf_dir}/{name}.wav.npy|{sid}" - }, - False: { - True: f"{gt_wavs_dir}/{name}.wav|{feature_dir}/{name}.npy|{energy_dir}/{name}.wav.npy|{sid}", - False: f"{gt_wavs_dir}/{name}.wav|{feature_dir}/{name}.npy|{sid}" - } - }[pitch_guidance][rms_extract] - - options.append(option) - - if include_mutes > 0: - mute_audio_path, mute_feature_path = ( - os.path.join(mute_base_path, "sliced_audios", f"mute{sample_rate}.wav"), - mute_file(embedders_mode, embedder_model, mute_base_path, rvc_version) - ) - - mute_f0_path, mute_f0nsf_path = ( - os.path.join(mute_base_path, 'f0', 'mute.wav.npy'), - os.path.join(mute_base_path, 'f0_voiced', 'mute.wav.npy') - ) - - mute_energy_path = os.path.join(mute_base_path, 'energy', 'mute.wav.npy') - - for sid in sids * include_mutes: - option = { - True: { - True: f"{mute_audio_path}|{mute_feature_path}|{mute_f0_path}|{mute_f0nsf_path}|{mute_energy_path}|{sid}", - False: f"{mute_audio_path}|{mute_feature_path}|{mute_f0_path}|{mute_f0nsf_path}|{sid}" - }, - False: { - True: f"{mute_audio_path}|{mute_feature_path}|{mute_energy_path}|{sid}", - False: f"{mute_audio_path}|{mute_feature_path}|{sid}" - } - }[pitch_guidance][rms_extract] - - options.append(option) - - shuffle(options) - with open(os.path.join(model_path, "filelist.txt"), "w") as f: - f.write("\n".join(options)) - - configs_path = os.path.join(model_path, "config.json") - with open(configs_path, "r", encoding="utf-8") as f: - data = json.load(f) - data["sid"] = len(sids) - - with open(configs_path, "w", encoding="utf-8") as f: - json.dump(data, f, indent=4) \ No newline at end of file diff --git a/main/inference/extracting/rms.py b/main/inference/extracting/rms.py deleted file mode 100644 index 643c5fe1cd9d29929b97697d21823251026651a9..0000000000000000000000000000000000000000 --- a/main/inference/extracting/rms.py +++ /dev/null @@ -1,114 +0,0 @@ -import os -import sys -import time -import tqdm -import torch -import traceback -import concurrent.futures - -import numpy as np -import torch.nn as nn - -sys.path.append(os.getcwd()) - -from main.library.audio.features import rms -from main.library.audio.audio import load_audio -from main.app.variables import logger, translations -from main.inference.extracting.setup_path import setup_paths - -class RMSEnergyExtractor(nn.Module): - def __init__( - self, - frame_length=2048, - hop_length=512, - center=True, - pad_mode = "reflect" - ): - super().__init__() - self.frame_length = frame_length - self.hop_length = hop_length - self.center = center - self.pad_mode = pad_mode - - def forward(self, x): - assert x.ndim == 2 - assert x.shape[0] == 1 - - if str(x.device).startswith(("ocl", "privateuseone")): x = x.contiguous() - - output = rms( - y=x.squeeze(0), - frame_length=self.frame_length, - hop_length=self.hop_length, - center=self.center, - pad_mode=self.pad_mode, - device=x.device - ) - - if str(x.device).startswith(("ocl", "privateuseone")): output = output.contiguous() - return output.squeeze(-2).to(x.device) - -def process_file_rms(files, device, threads): - threads = max(1, threads) - - module = RMSEnergyExtractor( - frame_length=2048, - hop_length=160, - center=True, - pad_mode = "reflect" - ).to(device).eval().float() - - def worker(file_info): - try: - file, out_path = file_info - out_file_path = os.path.join(out_path, os.path.basename(file)) - - if os.path.exists(out_file_path + ".npy"): return - feats = torch.from_numpy(load_audio(file, sample_rate=16000)).unsqueeze(0) - - with torch.no_grad(): - feats = module( - feats if device.startswith(("ocl", "privateuseone")) else feats.to(device) - ) - - np.save( - out_file_path, - feats.float().cpu().numpy(), - allow_pickle=False - ) - except: - logger.debug(traceback.format_exc()) - - with tqdm.tqdm(total=len(files), ncols=100, unit="p", leave=True) as pbar: - with concurrent.futures.ThreadPoolExecutor(max_workers=threads) as executor: - for _ in concurrent.futures.as_completed([executor.submit(worker, f) for f in files]): - pbar.update(1) - -def run_rms_extraction(exp_dir, num_processes, devices, rms_extract): - if rms_extract: - wav_path, out_path = setup_paths( - exp_dir, - rms_extract=rms_extract - ) - - paths = sorted([ - (os.path.join(wav_path, file), out_path) - for file in os.listdir(wav_path) - if file.endswith(".wav") - ]) - - start_time = time.time() - logger.info(translations["rms_start_extract"].format(num_processes=num_processes)) - - with concurrent.futures.ProcessPoolExecutor(max_workers=len(devices)) as executor: - concurrent.futures.wait([ - executor.submit( - process_file_rms, - paths[i::len(devices)], - devices[i], - num_processes // len(devices) - ) - for i in range(len(devices)) - ]) - - logger.info(translations["rms_success_extract"].format(elapsed_time=f"{(time.time() - start_time):.2f}")) \ No newline at end of file diff --git a/main/inference/extracting/setup_path.py b/main/inference/extracting/setup_path.py deleted file mode 100644 index cee59e54820c6449ad0fe96ef30542759a291089..0000000000000000000000000000000000000000 --- a/main/inference/extracting/setup_path.py +++ /dev/null @@ -1,26 +0,0 @@ -import os - -def setup_paths(exp_dir, version = None, rms_extract = False): - wav_path = os.path.join(exp_dir, "sliced_audios_16k") - - if rms_extract: - out_path = os.path.join(exp_dir, "energy") - os.makedirs(out_path, exist_ok=True) - - return wav_path, out_path - - if version: - out_path = os.path.join(exp_dir, f"{version}_extracted") - os.makedirs(out_path, exist_ok=True) - - return wav_path, out_path - else: - output_root1, output_root2 = ( - os.path.join(exp_dir, "f0"), - os.path.join(exp_dir, "f0_voiced") - ) - - os.makedirs(output_root1, exist_ok=True) - os.makedirs(output_root2, exist_ok=True) - - return wav_path, output_root1, output_root2 \ No newline at end of file diff --git a/main/inference/preprocess/preprocess.py b/main/inference/preprocess/preprocess.py deleted file mode 100644 index 670edc99b627df0d6c3c2bfd825975bc678cfc0c..0000000000000000000000000000000000000000 --- a/main/inference/preprocess/preprocess.py +++ /dev/null @@ -1,404 +0,0 @@ -import os -import sys -import time -import torch -import logging -import librosa -import argparse -import warnings - -import numpy as np -import torch.multiprocessing as mp - -from tqdm import tqdm -from scipy import signal -from scipy.io import wavfile -from concurrent.futures import ProcessPoolExecutor, as_completed - -sys.path.append(os.getcwd()) - -from main.library.utils import strtobool -from main.library.audio.audio import load_audio -from main.inference.preprocess.slicer2 import Slicer -from main.app.variables import config, logger, translations, configs, file_types - -if not config.debug_mode: - warnings.filterwarnings("ignore") - for l in ["numba.core.byteflow", "numba.core.ssa", "numba.core.interpreter"]: - logging.getLogger(l).setLevel(logging.ERROR) - -OVERLAP, MAX_AMPLITUDE, ALPHA, HIGH_PASS_CUTOFF, SAMPLE_RATE_16K = 0.3, 0.9, 0.75, 48, 16000 - -def parse_arguments(): - parser = argparse.ArgumentParser() - parser.add_argument("--preprocess", action='store_true') - parser.add_argument("--model_name", type=str, required=True) - parser.add_argument("--dataset_path", type=str, default="./dataset") - parser.add_argument("--sample_rate", type=int, required=True) - parser.add_argument("--cpu_cores", type=int, default=2) - parser.add_argument("--cut_preprocess", type=str, default="Automatic") - parser.add_argument("--process_effects", type=lambda x: bool(strtobool(x)), default=False) - parser.add_argument("--clean_dataset", type=lambda x: bool(strtobool(x)), default=False) - parser.add_argument("--clean_strength", type=float, default=0.7) - parser.add_argument("--chunk_len", type=float, default=3.0, required=False) - parser.add_argument("--overlap_len", type=float, default=0.3, required=False) - parser.add_argument("--normalization_mode", type=str, default="none", required=False) - parser.add_argument("--architecture", type=str, default="RVC") - - return parser.parse_args() - -class PreProcess: - def __init__( - self, - sr, - exp_dir, - per, - architecture = "RVC" - ): - if architecture == "RVC": slicer_params = {"sr": sr, "threshold": -42, "min_length": 1500, "min_interval": 400, "hop_size": 15, "max_sil_kept": 500} - else: slicer_params = {"sr": sr, "threshold": -40, "min_length": 7500, "min_interval": 100, "hop_size": 10, "max_sil_kept": 800} - - self.slicer = Slicer( - **slicer_params - ) - self.b_high, self.a_high = signal.butter( - N=5, - Wn=HIGH_PASS_CUTOFF, - btype="high", - fs=sr - ) - self.sr = sr - self.per = per - self.exp_dir = exp_dir - self.device = config.device - self.gt_wavs_dir = os.path.join(exp_dir, "sliced_audios") - self.wavs16k_dir = os.path.join(exp_dir, "sliced_audios_16k") - os.makedirs(self.gt_wavs_dir, exist_ok=True) - os.makedirs(self.wavs16k_dir, exist_ok=True) - - def _normalize_audio(self, audio): - tmp_max = np.abs(audio).max() - if tmp_max > 2.5: return None - return (audio / tmp_max * (MAX_AMPLITUDE * ALPHA)) + (1 - ALPHA) * audio - - def process_audio_segment(self, normalized_audio, sid, idx0, idx1, normalization_mode): - if normalized_audio is None: - logger.debug(f"{sid}-{idx0}-{idx1}-filtered") - return - - if normalization_mode == "post": normalized_audio = self._normalize_audio(normalized_audio) - - wavfile.write( - os.path.join(self.gt_wavs_dir, f"{sid}_{idx0}_{idx1}.wav"), - self.sr, - normalized_audio.astype(np.float32) - ) - - wavfile.write( - os.path.join(self.wavs16k_dir, f"{sid}_{idx0}_{idx1}.wav"), - SAMPLE_RATE_16K, - librosa.resample( - normalized_audio, - orig_sr=self.sr, - target_sr=SAMPLE_RATE_16K, - res_type="soxr_vhq" - ).astype(np.float32) - ) - - def simple_cut( - self, - audio, - sid, - idx0, - chunk_len, - overlap_len, - normalization_mode - ): - chunk_length = int(self.sr * chunk_len) - overlap_length = int(self.sr * overlap_len) - i = 0 - - while i < len(audio): - chunk = audio[i : i + chunk_length] - if normalization_mode == "post": chunk = self._normalize_audio(chunk) - - if len(chunk) == chunk_length: - wavfile.write( - os.path.join(self.gt_wavs_dir, f"{sid}_{idx0}_{i // (chunk_length - overlap_length)}.wav"), - self.sr, - chunk.astype(np.float32) - ) - - wavfile.write( - os.path.join(self.wavs16k_dir, f"{sid}_{idx0}_{i // (chunk_length - overlap_length)}.wav"), - SAMPLE_RATE_16K, - librosa.resample( - chunk, - orig_sr=self.sr, - target_sr=SAMPLE_RATE_16K, - res_type="soxr_vhq" - ).astype(np.float32) - ) - - i += chunk_length - overlap_length - - def process_audio( - self, - path, - idx0, - sid, - cut_preprocess, - process_effects, - clean_dataset, - clean_strength, - chunk_len, - overlap_len, - normalization_mode - ): - dataset_length = 0 - - try: - audio = load_audio(path, sample_rate=self.sr) - dataset_length = librosa.get_duration(y=audio, sr=self.sr) - - if process_effects: audio = signal.lfilter(self.b_high, self.a_high, audio) - if normalization_mode == "pre": audio = self._normalize_audio(audio) - - if clean_dataset: - if not hasattr(self, "tg"): - from main.library.audio.noisereduce import TorchGate - - self.tg = TorchGate( - self.sr, - prop_decrease=clean_strength - ).to(self.device) - - audio = self.tg( - torch.from_numpy(audio).unsqueeze(0).to(self.device).float() - ).squeeze(0).cpu().detach().numpy() - - if cut_preprocess == "Skip": - self.process_audio_segment( - audio, - sid, - idx0, - 0, - normalization_mode, - ) - elif cut_preprocess == "Simple": - self.simple_cut( - audio, - sid, - idx0, - chunk_len, - overlap_len, - normalization_mode, - ) - elif cut_preprocess == "Automatic": - idx1 = 0 - for audio_segment in self.slicer.slice(audio): - i = 0 - - while 1: - start = int(self.sr * (self.per - OVERLAP) * i) - i += 1 - - if len(audio_segment[start:]) > (self.per + OVERLAP) * self.sr: - self.process_audio_segment( - audio_segment[start : start + int(self.per * self.sr)], - sid, - idx0, - idx1, - normalization_mode - ) - - idx1 += 1 - else: - self.process_audio_segment( - audio_segment[start:], - sid, - idx0, - idx1, - normalization_mode - ) - - idx1 += 1 - break - except Exception as e: - raise RuntimeError(f"{translations['process_audio_error']}: {e}") - return dataset_length - -def format_duration(seconds): - return f"{int(seconds // 3600):02}:{int((seconds % 3600) // 60):02}:{int(seconds % 60):02}" - -def process_file(args): - ( - pp, - file, - cut_preprocess, - process_effects, - clean_dataset, - clean_strength, - chunk_len, - overlap_len, - normalization_mode - ) = args - - file_path, idx0, sid = file - - return pp.process_audio( - file_path, - idx0, - sid, - cut_preprocess, - process_effects, - clean_dataset, - clean_strength, - chunk_len, - overlap_len, - normalization_mode - ) - -def preprocess_training_set( - input_root, - sr, - num_processes, - exp_dir, - per, - cut_preprocess, - process_effects, - clean_dataset, - clean_strength, - chunk_len, - overlap_len, - normalization_mode, - architecture = "RVC" -): - start_time = time.time() - pp = PreProcess(sr, exp_dir, per, architecture) - logger.info(translations["start_preprocess"].format(num_processes=num_processes)) - dataset_length = 0 - files = [] - idx = 0 - - for root, _, filenames in os.walk(input_root): - try: - sid = 0 if root == input_root else int(os.path.basename(root)) - for f in filenames: - if f.lower().endswith(tuple(file_types)): - files.append((os.path.join(root, f), idx, sid)) - idx += 1 - except ValueError: - raise ValueError(f"{translations['not_integer']} '{os.path.basename(root)}'.") - - with tqdm(total=len(files), ncols=100, unit="f") as pbar: - with ProcessPoolExecutor(max_workers=num_processes) as executor: - futures = [ - executor.submit( - process_file, - ( - pp, - file, - cut_preprocess, - process_effects, - clean_dataset, - clean_strength, - chunk_len, - overlap_len, - normalization_mode - ) - ) - for file in files - ] - - for future in as_completed(futures): - try: - dataset_length += future.result() - except Exception as e: - raise RuntimeError(f"{translations['process_error']}: {e}") - pbar.update(1) - - elapsed_time = time.time() - start_time - logger.info(translations["dataset_duration"].format(duration=format_duration(dataset_length))) - logger.info(translations["preprocess_success"].format(elapsed_time=f"{elapsed_time:.2f}")) - -def main(): - args = parse_arguments() - experiment_directory = os.path.join(configs["logs_path"], args.model_name) - - num_processes = args.cpu_cores - num_processes = 2 if num_processes is None else int(num_processes) - - ( - dataset, - sample_rate, - cut_preprocess, - preprocess_effects, - clean_dataset, - clean_strength, - chunk_len, - overlap_len, - normalization_mode, - architecture - ) = ( - args.dataset_path, - args.sample_rate, - args.cut_preprocess, - args.process_effects, - args.clean_dataset, - args.clean_strength, - args.chunk_len, - args.overlap_len, - args.normalization_mode, - args.architecture - ) - - os.makedirs(experiment_directory, exist_ok=True) - - log_data = { - translations['modelname']: args.model_name, - translations['export_process']: experiment_directory, - translations['dataset_folder']: dataset, - translations['pretrain_sr']: sample_rate, - translations['cpu_core']: num_processes, - translations['split_audio']: cut_preprocess, - translations['preprocess_effect']: preprocess_effects, - translations['clear_audio']: clean_dataset, - translations['clean_strength']: clean_strength, - translations["architecture"]: architecture - } - - for key, value in log_data.items(): - logger.debug(f"{key}: {value}") - - pid_path = os.path.join(experiment_directory, "preprocess_pid.txt") - with open(pid_path, "w") as pid_file: - pid_file.write(str(os.getpid())) - - try: - preprocess_training_set( - dataset, - sample_rate, - num_processes, - experiment_directory, - config.per_preprocess, - cut_preprocess, - preprocess_effects, - clean_dataset, - clean_strength, - chunk_len, - overlap_len, - normalization_mode, - architecture - ) - except Exception as e: - logger.error(f"{translations['process_audio_error']} {e}") - import traceback - logger.debug(traceback.format_exc()) - - if os.path.exists(pid_path): os.remove(pid_path) - logger.info(f"{translations['preprocess_model_success']} {args.model_name}") - -if __name__ == "__main__": - mp.set_start_method("spawn", force=True) - main() \ No newline at end of file diff --git a/main/inference/preprocess/slicer2.py b/main/inference/preprocess/slicer2.py deleted file mode 100644 index 96b5338aa5d7417efa96f965a34987e87bc78d53..0000000000000000000000000000000000000000 --- a/main/inference/preprocess/slicer2.py +++ /dev/null @@ -1,266 +0,0 @@ -import numpy as np - -class Slicer: - def __init__( - self, - sr, - threshold = -40.0, - min_length = 5000, - min_interval = 300, - hop_size = 20, - max_sil_kept = 5000 - ): - min_interval = sr * min_interval / 1000 - self.threshold = 10 ** (threshold / 20.0) - self.hop_size = round(sr * hop_size / 1000) - self.win_size = min(round(min_interval), 4 * self.hop_size) - self.min_length = round(sr * min_length / 1000 / self.hop_size) - self.min_interval = round(min_interval / self.hop_size) - self.max_sil_kept = round(sr * max_sil_kept / 1000 / self.hop_size) - - def _apply_slice(self, waveform, begin, end): - start_idx = begin * self.hop_size - - return waveform[ - :, start_idx:min(waveform.shape[1], end * self.hop_size) - ] if len(waveform.shape) > 1 else waveform[ - start_idx:min(waveform.shape[0], end * self.hop_size) - ] - - def slice(self, waveform): - samples = waveform.mean(axis=0) if len(waveform.shape) > 1 else waveform - if samples.shape[0] <= self.min_length: return [waveform] - - rms_list = get_rms( - y=samples, - frame_length=self.win_size, - hop_length=self.hop_size - ).squeeze(0) - - sil_tags = [] - silence_start, clip_start = None, 0 - - for i, rms in enumerate(rms_list): - if rms < self.threshold: - if silence_start is None: silence_start = i - continue - - if silence_start is None: continue - is_leading_silence = silence_start == 0 and i > self.max_sil_kept - - need_slice_middle = ( - i - silence_start >= self.min_interval and i - clip_start >= self.min_length - ) - - if not is_leading_silence and not need_slice_middle: - silence_start = None - continue - - if i - silence_start <= self.max_sil_kept: - pos = rms_list[silence_start : i + 1].argmin() + silence_start - - sil_tags.append( - (0, pos) if silence_start == 0 else (pos, pos) - ) - - clip_start = pos - elif i - silence_start <= self.max_sil_kept * 2: - pos = rms_list[ - i - self.max_sil_kept : silence_start + self.max_sil_kept + 1 - ].argmin() - - pos += i - self.max_sil_kept - pos_r = rms_list[i - self.max_sil_kept : i + 1].argmin() + i - self.max_sil_kept - - if silence_start == 0: - sil_tags.append((0, pos_r)) - clip_start = pos_r - else: - sil_tags.append(( - min((rms_list[silence_start : silence_start + self.max_sil_kept + 1].argmin() + silence_start), pos), - max(pos_r, pos) - )) - - clip_start = max(pos_r, pos) - else: - pos_r = (rms_list[i - self.max_sil_kept : i + 1].argmin() + i - self.max_sil_kept) - - sil_tags.append( - ( - 0, - pos_r - ) if silence_start == 0 else ( - (rms_list[silence_start : silence_start + self.max_sil_kept + 1].argmin() + silence_start), - pos_r - ) - ) - - clip_start = pos_r - - silence_start = None - - total_frames = rms_list.shape[0] - - if ( - silence_start is not None and - total_frames - silence_start >= self.min_interval - ): - sil_tags.append(( - rms_list[silence_start : min(total_frames, silence_start + self.max_sil_kept) + 1].argmin() + silence_start, - total_frames + 1 - )) - - if not sil_tags: return [waveform] - else: - chunks = [] - if sil_tags[0][0] > 0: chunks.append(self._apply_slice(waveform, 0, sil_tags[0][0])) - - for i in range(len(sil_tags) - 1): - chunks.append(self._apply_slice(waveform, sil_tags[i][1], sil_tags[i + 1][0])) - - if sil_tags[-1][1] < total_frames: chunks.append(self._apply_slice(waveform, sil_tags[-1][1], total_frames)) - return chunks - -class Slicer2(Slicer): - def slice2(self, waveform): - samples = waveform.mean(axis=0) if len(waveform.shape) > 1 else waveform - - if samples.shape[0] <= self.min_length: - return [( - waveform, - 0, - samples.shape[0] - )] - - rms_list = get_rms( - y=samples, - frame_length=self.win_size, - hop_length=self.hop_size - ).squeeze(0) - - sil_tags = [] - silence_start, clip_start = None, 0 - - for i, rms in enumerate(rms_list): - if rms < self.threshold: - if silence_start is None: silence_start = i - continue - - if silence_start is None: continue - - is_leading_silence = silence_start == 0 and i > self.max_sil_kept - - need_slice_middle = ( - i - silence_start >= self.min_interval and i - clip_start >= self.min_length - ) - - if not is_leading_silence and not need_slice_middle: - silence_start = None - continue - - if i - silence_start <= self.max_sil_kept: - pos = rms_list[silence_start : i + 1].argmin() + silence_start - sil_tags.append((0, pos) if silence_start == 0 else (pos, pos)) - - clip_start = pos - elif i - silence_start <= self.max_sil_kept * 2: - pos = rms_list[ - i - self.max_sil_kept : silence_start + self.max_sil_kept + 1 - ].argmin() - - pos += i - self.max_sil_kept - pos_r = rms_list[i - self.max_sil_kept : i + 1].argmin() + i - self.max_sil_kept - - if silence_start == 0: - sil_tags.append((0, pos_r)) - clip_start = pos_r - else: - sil_tags.append(( - min((rms_list[silence_start : silence_start + self.max_sil_kept + 1].argmin() + silence_start), pos), - max(pos_r, pos) - )) - - clip_start = max(pos_r, pos) - else: - pos_r = rms_list[i - self.max_sil_kept : i + 1].argmin() + i - self.max_sil_kept - - sil_tags.append( - ( - 0, - pos_r - ) if silence_start == 0 else ( - (rms_list[silence_start : silence_start + self.max_sil_kept + 1].argmin() + silence_start), - pos_r - ) - ) - - clip_start = pos_r - - silence_start = None - - total_frames = rms_list.shape[0] - if (silence_start is not None and total_frames - silence_start >= self.min_interval): - sil_tags.append(( - rms_list[silence_start : min(total_frames, silence_start + self.max_sil_kept) + 1].argmin() + silence_start, - total_frames + 1 - )) - - if not sil_tags: - return [( - waveform, - 0, - samples.shape[-1] - )] - else: - chunks = [] - - if sil_tags[0][0] > 0: - chunks.append(( - self._apply_slice(waveform, 0, sil_tags[0][0]), - 0, - sil_tags[0][0] * self.hop_size - )) - - for i in range(len(sil_tags) - 1): - chunks.append(( - self._apply_slice(waveform, sil_tags[i][1], sil_tags[i + 1][0]), - sil_tags[i][1] * self.hop_size, - sil_tags[i + 1][0] * self.hop_size - )) - - if sil_tags[-1][1] < total_frames: - chunks.append(( - self._apply_slice(waveform, sil_tags[-1][1], total_frames), - sil_tags[-1][1] * self.hop_size, - samples.shape[-1] - )) - - return chunks - -def get_rms(y, frame_length=2048, hop_length=512, pad_mode="constant"): - y = np.pad(y, (int(frame_length // 2), int(frame_length // 2)), mode=pad_mode) - axis = -1 - - x_shape_trimmed = list(y.shape) - x_shape_trimmed[axis] -= frame_length - 1 - - xw = np.moveaxis( - np.lib.stride_tricks.as_strided( - y, - shape=tuple(x_shape_trimmed) + tuple([frame_length]), - strides=y.strides + tuple([y.strides[axis]]) - ), - -1, - axis - 1 if axis < 0 else axis + 1 - ) - - slices = [slice(None)] * xw.ndim - slices[axis] = slice(0, None, hop_length) - - return np.sqrt( - np.mean( - np.abs(xw[tuple(slices)]) ** 2, - axis=-2, - keepdims=True - ) - ) \ No newline at end of file diff --git a/main/inference/realtime/audio.py b/main/inference/realtime/audio.py deleted file mode 100644 index 1c7180bdb691a5447ddf4fb41247ef8523d37438..0000000000000000000000000000000000000000 --- a/main/inference/realtime/audio.py +++ /dev/null @@ -1,481 +0,0 @@ -import os -import sys -import soxr -import traceback - -import numpy as np - -from queue import Queue - -sys.path.append(os.getcwd()) - -import main.library.audio.sounddevice as sd - -from main.app.variables import logger, translations - -class ServerAudioDevice: - def __init__( - self, - index, - name, - host_api, - max_input_channels, - max_output_channels, - default_samplerate - ): - self.index = index - self.name = name - self.host_api = host_api - self.max_input_channels = max_input_channels - self.max_output_channels = max_output_channels - self.default_samplerate = default_samplerate - -def check_the_device(device, type = "input"): - if sys.platform == "linux" and "hw:" not in device["name"]: - return False - - stream_device = ( - sd.InputStream if type == "input" else sd.OutputStream - ) - - try: - with stream_device( - device=device["index"], - dtype=np.float32, - samplerate=device["default_samplerate"] - ): - return True - except Exception: - return False - -def list_audio_device(): - try: - audio_device_list = sd.query_devices() - except Exception as e: - logger.error(translations["error_occurred"].format(e=e)) - audio_device_list = [] - except OSError as e: - logger.debug(translations["error_occurred"].format(e=e)) - audio_device_list = [] - - input_audio_device_list = [ - d for d in audio_device_list - if d["max_input_channels"] > 0 and check_the_device(d, "input") - ] - - output_audio_device_list = [ - d for d in audio_device_list - if d["max_output_channels"] > 0 and check_the_device(d, "output") - ] - - try: - hostapis = sd.query_hostapis() - except Exception as e: - logger.error(translations["error_occurred"].format(e=e)) - hostapis = [] - except OSError as e: - logger.debug(translations["error_occurred"].format(e=e)) - hostapis = [] - - audio_input_device, audio_output_device = [], [] - - for d in input_audio_device_list: - input_audio_device = ServerAudioDevice( - index=d["index"], - name=d["name"], - host_api=hostapis[d["hostapi"]]["name"], - max_input_channels=d["max_input_channels"], - max_output_channels=d["max_output_channels"], - default_samplerate=d["default_samplerate"], - ) - audio_input_device.append(input_audio_device) - - for d in output_audio_device_list: - output_audio_device = ServerAudioDevice( - index=d["index"], - name=d["name"], - host_api=hostapis[d["hostapi"]]["name"], - max_input_channels=d["max_input_channels"], - max_output_channels=d["max_output_channels"], - default_samplerate=d["default_samplerate"], - ) - audio_output_device.append(output_audio_device) - - return audio_input_device, audio_output_device - -def audio_device(): - try: - input_devices, output_devices = list_audio_device() - - def priority(name): - n = name.lower() - - if "virtual" in n: - return 0 - if "vb" in n: - return 1 - - return 2 - - output_sorted = sorted( - output_devices, - key=lambda d: priority(d.name) - ) - input_sorted = sorted( - input_devices, key=lambda d: priority(d.name), reverse=True - ) - - input_device_list = { - f"{input_sorted.index(d)+1}: {d.name} ({d.host_api})": [d.index, d.max_input_channels] for d in input_sorted - } - output_device_list = { - f"{output_sorted.index(d)+1}: {d.name} ({d.host_api})": [d.index, d.max_output_channels] for d in output_sorted - } - - return input_device_list, output_device_list - except Exception: - return {}, {} - -class Audio: - def __init__( - self, - callbacks, - f0_up_key = 0, - index_rate = 0.5, - protect = 0.5, - filter_radius = 3, - rms_mix_rate = 1, - f0_autotune = False, - f0_autotune_strength = 1, - proposal_pitch = False, - proposal_pitch_threshold = 255.0, - input_audio_gain = 1.0, - output_audio_gain = 1.0, - monitor_audio_gain = 1.0, - monitor = False, - embedders_mix = False, - embedders_mix_layers = 9, - embedders_mix_ratio = 0.5 - ): - self.callbacks = callbacks - self.mon_queue = Queue() - self.performance = [0, 0, 0] - self.stream = None - self.input_stream = None - self.output_stream = None - self.monitor = None - self.running = False - self.input_audio_gain = input_audio_gain - self.output_audio_gain = output_audio_gain - self.monitor_audio_gain = monitor_audio_gain - self.use_monitor = monitor - self.f0_up_key = f0_up_key - self.index_rate = index_rate - self.protect = protect - self.filter_radius = filter_radius - self.rms_mix_rate = rms_mix_rate - self.f0_autotune = f0_autotune - self.f0_autotune_strength = f0_autotune_strength - self.proposal_pitch = proposal_pitch - self.proposal_pitch_threshold = proposal_pitch_threshold - self.embedders_mix = embedders_mix - self.embedders_mix_layers = embedders_mix_layers - self.embedders_mix_ratio = embedders_mix_ratio - - def get_audio_device(self, input_index = None, output_index = None): - audioinput, audiooutput = list_audio_device() - - inputs, outputs = ( - [ - x for x in audioinput - if x.index == input_index - ] if input_index is not None else [], - [ - x for x in audiooutput - if x.index == output_index - ] if output_index is not None else [] - ) - - return (inputs[0] if len(inputs) > 0 else None), (outputs[0] if len(outputs) > 0 else None) - - def process_data_with_time(self, indata): - indata = indata * self.input_audio_gain - unpacked_data = np.mean(indata, axis=1) if indata.shape[1] > 1 else indata.flatten() - - out_wav, vol, perf, _ = self.callbacks.change_voice( - unpacked_data, - self.f0_up_key, - self.index_rate, - self.protect, - self.filter_radius, - self.rms_mix_rate, - self.f0_autotune, - self.f0_autotune_strength, - self.proposal_pitch, - self.proposal_pitch_threshold, - self.embedders_mix, - self.embedders_mix_layers, - self.embedders_mix_ratio - ) - - self.performance = perf[1] - self.volume = vol - return out_wav - - def audio_stream_no_output_callback(self, indata, frames, times, status): - try: - out_wav = self.process_data_with_time(indata) - self.mon_queue.put(out_wav) - except Exception as e: - logger.error(translations["error_occurred"].format(e=e)) - logger.debug(traceback.format_exc()) - - def audio_stream_callback(self, indata, outdata, frames, times, status): - try: - if not self.running: - outdata_fill = np.zeros_like(outdata) - outdata[:] = outdata_fill - return - - out_wav = self.process_data_with_time(indata) - output_channels = outdata.shape[1] - if self.use_monitor: self.mon_queue.put(out_wav) - - outdata[:] = ( - np.repeat(out_wav, output_channels).reshape(-1, output_channels) - * self.output_audio_gain - ) - except Exception as e: - logger.error(translations["error_occurred"].format(e=e)) - logger.debug(traceback.format_exc()) - - def audio_queue(self, outdata, gain, sample_rate, sample_rate_out = None): - try: - if not self.running: - outdata_fill = np.zeros_like(outdata) - outdata[:] = outdata_fill - return - - mon_wav = self.mon_queue.get() - - while self.mon_queue.qsize() > 0: - self.mon_queue.get() - - if sample_rate != sample_rate_out: mon_wav = soxr.resample(mon_wav, sample_rate, sample_rate_out) - output_channels = outdata.shape[1] - - outdata[:] = ( - np.repeat(mon_wav, output_channels).reshape(-1, output_channels) * gain - ) - except Exception as e: - logger.error(translations["error_occurred"].format(e=e)) - logger.debug(traceback.format_exc()) - - def run_audio_stream( - self, - block_frame, - input_device_id, - output_device_id, - output_monitor_id, - input_audio_sample_rate, - output_audio_sample_rate, - output_monitor_sample_rate, - input_max_channel, - output_max_channel, - output_monitor_max_channel, - input_extra_setting, - output_extra_setting, - output_monitor_extra_setting, - use_asio = False - ): - if use_asio: - try: - sd.terminate() - sd.initialize() - except: - pass - - if input_device_id != output_device_id or input_audio_sample_rate != output_audio_sample_rate: - self.input_stream = sd.InputStream( - callback=self.audio_stream_no_output_callback, - latency="low", - dtype=np.float32, - device=input_device_id, - blocksize=block_frame, - samplerate=input_audio_sample_rate, - channels=input_max_channel, - extra_settings=input_extra_setting - ) - self.output_stream = sd.OutputStream( - callback=lambda outdata, frames, times, status: self.audio_queue(outdata, self.output_audio_gain, input_audio_sample_rate, output_audio_sample_rate), - latency="low", - dtype=np.float32, - device=output_device_id, - blocksize=int(block_frame / input_audio_sample_rate * output_audio_sample_rate) if input_audio_sample_rate != output_audio_sample_rate else block_frame, - samplerate=output_audio_sample_rate, - channels=output_max_channel, - extra_settings=output_extra_setting - ) - - self.input_stream.start() - self.output_stream.start() - else: - self.stream = sd.Stream( - callback=self.audio_stream_callback, - latency="low", - dtype=np.float32, - device=(input_device_id, output_device_id), - blocksize=block_frame, - samplerate=input_audio_sample_rate, - channels=(input_max_channel, output_max_channel), - extra_settings=(input_extra_setting, output_extra_setting), - ) - - self.stream.start() - - if self.use_monitor: - self.monitor = sd.OutputStream( - callback=lambda outdata, frames, times, status: self.audio_queue(outdata, self.monitor_audio_gain, input_audio_sample_rate, output_monitor_sample_rate), - latency="low", - dtype=np.float32, - device=output_monitor_id, - blocksize=int(block_frame / input_audio_sample_rate * output_monitor_sample_rate) if input_audio_sample_rate != output_monitor_sample_rate else block_frame, - samplerate=output_monitor_sample_rate, - channels=output_monitor_max_channel, - extra_settings=output_monitor_extra_setting - ) - self.monitor.start() - - def stop(self): - self.running = False - - if self.stream is not None: - self.stream.stop() - self.stream.close() - self.stream = None - - if self.input_stream is not None: - self.input_stream.stop() - self.input_stream.close() - self.input_stream = None - - if self.output_stream is not None: - self.output_stream.stop() - self.output_stream.close() - self.output_stream = None - - if self.monitor is not None: - self.monitor.stop() - self.monitor.close() - self.monitor = None - - def start( - self, - input_device_id, - output_device_id, - output_monitor_id, - exclusive_mode, - asio_input_channel, - asio_output_channel, - asio_output_monitor_channel, - read_chunk_size, - input_audio_sample_rate, - output_audio_sample_rate, - output_monitor_sample_rate, - asio_output_stereo = True, - asio_monitor_stereo = True - ): - self.stop() - input_audio_device, output_audio_device = self.get_audio_device(input_device_id, output_device_id) - - input_channels, output_channels = ( - input_audio_device.max_input_channels, - output_audio_device.max_output_channels - ) - - use_asio = False - input_extra_setting, output_extra_setting = None, None - output_monitor_extra_setting, monitor_channels = None, None - - if ( - input_audio_device and - "WASAPI" in input_audio_device.host_api - ): - input_extra_setting = sd.WasapiSettings( - exclusive=exclusive_mode, - auto_convert=not exclusive_mode - ) - elif ( - input_audio_device and - "ASIO" in input_audio_device.host_api and - asio_input_channel != -1 - ): - input_extra_setting = sd.AsioSettings( - channel_selectors=[asio_input_channel] - ) - input_channels = 1 - use_asio = True - - if ( - output_audio_device and - "WASAPI" in output_audio_device.host_api - ): - output_extra_setting = sd.WasapiSettings( - exclusive=exclusive_mode, - auto_convert=not exclusive_mode - ) - elif ( - input_audio_device and - "ASIO" in input_audio_device.host_api and - asio_output_channel != -1 - ): - output_selectors = [asio_output_channel, asio_output_channel + 1] if asio_output_stereo else [asio_output_channel] - output_extra_setting = sd.AsioSettings(channel_selectors=output_selectors) - output_channels = len(output_selectors) - use_asio = True - - if self.use_monitor: - _, output_monitor_device = self.get_audio_device(output_index=output_monitor_id) - monitor_channels = output_monitor_device.max_output_channels - - if ( - output_monitor_device and - "WASAPI" in output_monitor_device.host_api - ): - output_monitor_extra_setting = sd.WasapiSettings( - exclusive=exclusive_mode, - auto_convert=not exclusive_mode - ) - elif ( - output_monitor_device and - "ASIO" in output_monitor_device.host_api and - asio_output_monitor_channel != -1 - ): - monitor_selectors = [output_monitor_device, output_monitor_device + 1] if asio_monitor_stereo else [output_monitor_device] - output_monitor_extra_setting = sd.AsioSettings(channel_selectors=monitor_selectors) - monitor_channels = len(monitor_selectors) - use_asio = True - - block_frame = read_chunk_size * 128 - - try: - self.run_audio_stream( - block_frame, - input_device_id, - output_device_id, - output_monitor_id, - input_audio_sample_rate, - output_audio_sample_rate, - output_monitor_sample_rate, - input_channels, - output_channels, - monitor_channels, - input_extra_setting, - output_extra_setting, - output_monitor_extra_setting, - use_asio=use_asio - ) - self.running = True - except Exception as e: - logger.error(translations["error_occurred"].format(e=e)) - logger.debug(traceback.format_exc()) \ No newline at end of file diff --git a/main/inference/realtime/callbacks.py b/main/inference/realtime/callbacks.py deleted file mode 100644 index 9901379b41a63507ccb6492c5192c9752b57056d..0000000000000000000000000000000000000000 --- a/main/inference/realtime/callbacks.py +++ /dev/null @@ -1,159 +0,0 @@ -import os -import sys -import threading - -import numpy as np - -sys.path.append(os.getcwd()) - -from main.inference.realtime.audio import Audio -from main.app.variables import logger, translations -from main.inference.realtime.realtime import VoiceChanger - -class AudioCallbacks: - def __init__( - self, - pass_through = False, - read_chunk_size = 192, - cross_fade_overlap_size = 0.1, - input_sample_rate = 48000, - output_sample_rate = 48000, - extra_convert_size = 0.5, - model_path = None, - index_path = None, - f0_method = "rmvpe", - predictor_onnx = False, - embedder_model = "hubert_base", - embedders_mode = "fairseq", - sample_rate = 16000, - hop_length = 160, - silent_threshold = -90, - f0_up_key = 0, - index_rate = 0.5, - protect = 0.5, - filter_radius = 3, - rms_mix_rate = 1, - f0_autotune = False, - f0_autotune_strength = 1, - proposal_pitch = False, - proposal_pitch_threshold = 255.0, - input_audio_gain = 1.0, - output_audio_gain = 1.0, - monitor_audio_gain = 1.0, - monitor = False, - vad_enabled = False, - vad_sensitivity = 3, - vad_frame_ms = 30, - clean_audio = False, - clean_strength = 0.7, - post_process = False, - sid = 0, - noise_scale = 0.35, - nprobe = 1, - record_audio = False, - record_audio_path = None, - export_format = "WAV", - embedders_mix = False, - embedders_mix_layers = 9, - embedders_mix_ratio = 0.5, - **kwargs - ): - self.pass_through = pass_through - self.input_sample_rate = input_sample_rate - self.output_sample_rate = output_sample_rate - self.lock = threading.Lock() - self.vc = VoiceChanger( - read_chunk_size, - cross_fade_overlap_size, - input_sample_rate, - output_sample_rate, - extra_convert_size, - model_path, - index_path, - f0_method, - predictor_onnx, - embedder_model, - embedders_mode, - sample_rate, - hop_length, - silent_threshold, - vad_enabled, - vad_sensitivity, - vad_frame_ms, - clean_audio, - clean_strength, - post_process, - sid, - noise_scale, - nprobe, - record_audio, - record_audio_path, - export_format, - **kwargs - ) - self.audio = Audio( - self, - f0_up_key, - index_rate, - protect, - filter_radius, - rms_mix_rate, - f0_autotune, - f0_autotune_strength, - proposal_pitch, - proposal_pitch_threshold, - input_audio_gain, - output_audio_gain, - monitor_audio_gain, - monitor, - embedders_mix, - embedders_mix_layers, - embedders_mix_ratio - ) - - def change_voice( - self, - received_data, - f0_up_key = 0, - index_rate = 0.5, - protect = 0.5, - filter_radius = 3, - rms_mix_rate = 1, - f0_autotune = False, - f0_autotune_strength = 1, - proposal_pitch = False, - proposal_pitch_threshold = 255.0, - embedders_mix = False, - embedders_mix_layers = 9, - embedders_mix_ratio = 0.5 - ): - if self.pass_through: - vol = float(np.sqrt(np.square(received_data).mean(dtype=np.float32))) - return received_data, vol, [0, 0, 0], None - - try: - with self.lock: - audio, vol, perf = self.vc.on_request( - received_data, - f0_up_key, - index_rate, - protect, - filter_radius, - rms_mix_rate, - f0_autotune, - f0_autotune_strength, - proposal_pitch, - proposal_pitch_threshold, - embedders_mix, - embedders_mix_layers, - embedders_mix_ratio - ) - - return audio, vol, perf, None - except RuntimeError as e: - import traceback - - logger.debug(traceback.format_exc()) - logger.error(translations["error_occurred"].format(e=e)) - - return np.zeros(1, dtype=np.float32), 0, [0, 0, 0], None \ No newline at end of file diff --git a/main/inference/realtime/client.py b/main/inference/realtime/client.py deleted file mode 100644 index 1181ef6dc67bc6181ba32e447757f05f058a3e46..0000000000000000000000000000000000000000 --- a/main/inference/realtime/client.py +++ /dev/null @@ -1,350 +0,0 @@ -import os -import sys -import json - -import numpy as np - -from fastapi import FastAPI, WebSocketDisconnect, WebSocket, Request - -sys.path.append(os.getcwd()) - -from main.library.utils import clear_gpu_cache -from main.inference.realtime.realtime import VoiceChanger -from main.app.variables import configs, translations, logger, config - -app = FastAPI() -vc_instance = None -params = {} - -PIPELINE_SAMPLE_RATE = 16000 -DEVICE_SAMPLE_RATE = 48000 - -@app.websocket("/change-config") -async def change_config(ws: WebSocket): - global vc_instance, params - - await ws.accept() - if vc_instance is None: return - - text = await ws.receive_text() - jsons = json.loads(text) - - if jsons["if_kwargs"]: - params["kwargs"][jsons["key"]] = jsons["value"] - else: - params[jsons["key"]] = jsons["value"] - - crossfade_frame = int(params.get("cross_fade_overlap_size", 0.1) * DEVICE_SAMPLE_RATE) - extra_frame = int(params.get("extra_convert_size", 0.5) * DEVICE_SAMPLE_RATE) - - if ( - vc_instance.crossfade_frame != crossfade_frame or - vc_instance.extra_frame != extra_frame - ): - del ( - vc_instance.fade_in_window, - vc_instance.fade_out_window, - vc_instance.sola_buffer - ) - - vc_instance.vc_model.realloc( - vc_instance.block_frame, - vc_instance.extra_frame, - vc_instance.crossfade_frame, - vc_instance.sola_search_frame, - ) - vc_instance.generate_strength() - - vc_instance.vc_model.input_sensitivity = 10 ** (params.get("silent_threshold", -90) / 20) - - vad_enabled = params.get("vad_enabled", True) - sensitivity_mode = params.get("vad_sensitivity", 3) - vad_frame_ms = params.get("vad_frame_ms", 30) - - if vad_enabled is False: - vc_instance.vc_model.vad = None - elif vad_enabled and vc_instance.vc_model.vad is None: - from main.inference.realtime.vad_utils import VADProcessor - - vc_instance.vc_model.vad = VADProcessor( - sensitivity_mode=sensitivity_mode, - sample_rate=vc_instance.vc_model.sample_rate, - frame_duration_ms=vad_frame_ms - ) - - if vc_instance.vc_model.vad is not None: - vc_instance.vc_model.vad.vad.set_mode(sensitivity_mode) - vc_instance.vc_model.vad.frame_length = int(vc_instance.vc_model.sample_rate * (vad_frame_ms / 1000.0)) - - clean_audio = params.get("clean_audio", False) - clean_strength = params.get("clean_strength", 0.5) - - if clean_audio is False: - vc_instance.vc_model.tg = None - elif clean_audio and vc_instance.vc_model.tg is None: - from main.library.audio.noisereduce import TorchGate - - vc_instance.vc_model.tg = ( - TorchGate( - vc_instance.vc_model.pipeline.tgt_sr, - prop_decrease=clean_strength, - ).to(config.device) - ) - - if vc_instance.vc_model.tg is not None: - vc_instance.vc_model.tg.prop_decrease = clean_strength - - post_process = params.get("post_process", False) - kwargs = params.get("kwargs", {}) - - if post_process is False: - vc_instance.vc_model.board = None - vc_instance.vc_model.kwargs = None - elif post_process and vc_instance.vc_model.kwargs != kwargs: - new_board = vc_instance.vc_model.setup_pedalboard(**kwargs) - vc_instance.vc_model.board = new_board - vc_instance.vc_model.kwargs = kwargs.copy() - - noise_scale = params.get("noise_scale", 0.35) - model_pth = params.get("model_path", vc_instance.vc_model.model_path) - model_pth = os.path.join(configs["weights_path"], model_pth) if not os.path.exists(model_pth) else model_pth - - if model_pth and vc_instance.vc_model.model_path != model_pth: - vc_instance.vc_model.model_path = model_pth - vc_instance.vc_model.pipeline.vc.setup(model_pth, noise_scale) - vc_instance.vc_model.pipeline.version = vc_instance.vc_model.pipeline.vc.version - vc_instance.vc_model.pipeline.energy = vc_instance.vc_model.pipeline.vc.energy - vc_instance.vc_model.pipeline.rms = vc_instance.vc_model.pipeline.setup_rms() if vc_instance.vc_model.pipeline.vc.energy else None - - sid = params.get("sid", vc_instance.vc_model.pipeline.sid) - if vc_instance.vc_model.pipeline.sid != sid: - import torch - vc_instance.vc_model.pipeline.torch_sid = torch.tensor( - [sid], device=vc_instance.vc_model.pipeline.device, dtype=torch.int64 - ) - - index_path = params.get("index_path", None) - if index_path: - if vc_instance.vc_model.index_path != index_path: - from main.library.utils import load_faiss_index - - nprobe = params.get("nprobe", 1) - index, index_reconstruct = load_faiss_index( - index_path.strip().strip('"').strip("\n").strip('"').strip().replace("trained", "added") - ) - - vc_instance.vc_model.pipeline.index = index - if vc_instance.vc_model.pipeline.index.index is not None: vc_instance.vc_model.pipeline.index.index.nprobe = nprobe - vc_instance.vc_model.pipeline.big_tsr = index_reconstruct - vc_instance.vc_model.index_path = index_path - else: - vc_instance.vc_model.pipeline.index = None - vc_instance.vc_model.pipeline.big_tsr = None - vc_instance.vc_model.index_path = None - - f0_method = params.get("f0_method", vc_instance.vc_model.pipeline.f0_method) - predictor_onnx = params.get("predictor_onnx", vc_instance.vc_model.pipeline.predictor.predictor_onnx) - embedders = params.get("embedder_model", vc_instance.vc_model.embedder_model) - embedders_mode = params.get("embedders_mode", vc_instance.vc_model.embedders_mode) - custom_embedders = params.get("embedder_model_custom", None) - embedder_model = (embedders if embedders != "custom" else custom_embedders) - - from main.library.utils import check_assets - check_assets(f0_method, embedders, predictor_onnx, embedders_mode) - - if ( - vc_instance.vc_model.pipeline.f0_method != f0_method or - vc_instance.vc_model.pipeline.predictor.predictor_onnx != predictor_onnx - ): - old_predictor = vc_instance.vc_model.pipeline.predictor - del old_predictor - - vc_instance.vc_model.pipeline.predictor = vc_instance.vc_model.pipeline.setup_predictor(PIPELINE_SAMPLE_RATE, params.get("hop_length", vc_instance.vc_model.pipeline.predictor.hop_length), predictor_onnx) - vc_instance.vc_model.pipeline.f0_method = f0_method - vc_instance.vc_model.pipeline.predictor.predictor_onnx = predictor_onnx - - if embedder_model: - if ( - vc_instance.vc_model.embedder_model != embedder_model or - vc_instance.vc_model.embedders_mode != embedders_mode - ): - old_embedder = vc_instance.vc_model.pipeline.embedder - del old_embedder - - - vc_instance.vc_model.pipeline.embedder = vc_instance.vc_model.pipeline.setup_embedder(embedder_model, embedders_mode) - vc_instance.vc_model.embedder_model = embedder_model - vc_instance.vc_model.embedders_mode = embedders_mode - - if ( - vc_instance.vc_model.pipeline.vc.architecture == "SVC" and - vc_instance.vc_model.pipeline.vc.net_g.noise_scale != noise_scale - ): - vc_instance.vc_model.pipeline.vc.net_g.noise_scale = noise_scale - -@app.post("/record") -async def record(request: Request): - global vc_instance - - data = await request.json() - record_button = data.get("record_button", translations["stop_record"]) - record_audio_path = data.get("record_audio_path", None) - export_format = data.get("export_format", "wav") - - if vc_instance is None: - return { - "type": "warnings", - "value": translations["realtime_not_found"], - "button": translations["start_record"], - "path": None - } - - if record_button == translations["start_record"]: - if not record_audio_path: - record_audio_path = os.path.join(configs["audios_path"], "record_audio.wav") - - vc_instance.record_audio = True - vc_instance.record_audio_path = record_audio_path - vc_instance.export_format = export_format - vc_instance.setup_soundfile_record() - - return { - "type": "info", - "value": translations["starting_record"], - "button": translations["stop_record"], - "path": None - } - else: - vc_instance.record_audio = False - vc_instance.record_audio_path = None - vc_instance.soundfile = None - - return { - "type": "info", - "value": translations["stopping_record"], - "button": translations["start_record"], - "path": record_audio_path - } - -@app.websocket("/ws-audio") -async def websocket_audio(ws: WebSocket): - global vc_instance, params - await ws.accept() - - logger.info(translations["ws_connected"]) - - try: - text = await ws.receive_text() - params = json.loads(text) - - read_chunk_size = int(params["chunk_size"]) - block_frame = read_chunk_size * 128 - embedders = params["embedders"] - - model_pth = params["model_pth"] - model_pth = os.path.join(configs["weights_path"], model_pth) if not os.path.exists(model_pth) else model_pth - - if ( - not model_pth or - not os.path.exists(model_pth) or - os.path.isdir(model_pth) or - not model_pth.endswith((".pth", ".onnx")) - ): - logger.warning(translations["provide_file"].format(filename=translations["model"])) - await ws.send_text(json.dumps({"type": "warnings", "value": translations["provide_file"].format(filename=translations["model"])})) - return - - logger.info(translations["start_realtime"]) - - if vc_instance is None: - vc_instance = VoiceChanger( - read_chunk_size=read_chunk_size, - cross_fade_overlap_size=params["cross_fade_overlap_size"], - input_sample_rate=DEVICE_SAMPLE_RATE, - output_sample_rate=DEVICE_SAMPLE_RATE, - extra_convert_size=params["extra_convert_size"], - model_path=model_pth, - index_path=params["model_index"], - f0_method=params["f0_method"], - predictor_onnx=params["predictor_onnx"], - embedder_model=(embedders if embedders != "custom" else params["custom_embedders"]), - embedders_mode=params["embedders_mode"], - sample_rate=PIPELINE_SAMPLE_RATE, - hop_length=params["hop_length"], - silent_threshold=params["silent_threshold"], - vad_enabled=params["vad_enabled"], - vad_sensitivity=params["vad_sensitivity"], - vad_frame_ms=params["vad_frame_ms"], - clean_audio=params["clean_audio"], - clean_strength=params["clean_strength"], - post_process=params["post_process"], - sid=params["sid"], - noise_scale=params["noise_scale"], - nprobe=params["nprobe"], - record_audio=False, - record_audio_path=None, - export_format="wav", - **params["kwargs"] - ) - - noise_scale = params["noise_scale"] - if ( - vc_instance is not None and - vc_instance.vc_model.pipeline.vc.architecture == "SVC" and - vc_instance.vc_model.pipeline.vc.net_g.noise_scale != noise_scale - ): - vc_instance.vc_model.pipeline.vc.net_g.noise_scale = noise_scale - - logger.info(translations["realtime_is_ready"]) - - while 1: - audio = await ws.receive_bytes() - arr = np.frombuffer(audio, dtype=np.float32) - - if arr.size != block_frame: - arr = ( - np.pad(arr, (0, block_frame - arr.size)).astype(np.float32) - if arr.size < block_frame else - arr[:block_frame].astype(np.float32) - ) - - if vc_instance is None: return - - audio_output, vol, perf = vc_instance.on_request( - arr * (params["input_audio_gain"] / 100.0), - f0_up_key=params["f0_up_key"], - index_rate=params["index_rate"], - protect=params["protect"], - filter_radius=params["filter_radius"], - rms_mix_rate=params["rms_mix_rate"], - f0_autotune=params["f0_autotune"], - f0_autotune_strength=params["f0_autotune_strength"], - proposal_pitch=params["proposal_pitch"], - proposal_pitch_threshold=params["proposal_pitch_threshold"], - embedders_mix=params["embedders_mix"], - embedders_mix_layers=params["embedders_mix_layers"], - embedders_mix_ratio=params["embedders_mix_ratio"], - ) - - await ws.send_text(json.dumps({"type": "latency", "value": perf[1], "volume": vol})) - await ws.send_bytes(audio_output.tobytes()) - except WebSocketDisconnect: - logger.info(translations["ws_disconnected"]) - except Exception as e: - import traceback - - logger.debug(traceback.format_exc()) - logger.info(translations["error_occurred"].format(e=e)) - finally: - if vc_instance is not None: - del vc_instance - vc_instance = None - - clear_gpu_cache() - - try: - await ws.close() - except: - pass - - logger.info(translations["ws_closed"]) \ No newline at end of file diff --git a/main/inference/realtime/pipeline.py b/main/inference/realtime/pipeline.py deleted file mode 100644 index 6be624161a040696c1853b1aab3f29aab7c3ed63..0000000000000000000000000000000000000000 --- a/main/inference/realtime/pipeline.py +++ /dev/null @@ -1,345 +0,0 @@ -import os -import sys -import torch - -import numpy as np -import torch.nn.functional as F -import torchaudio.transforms as tat - -sys.path.append(os.getcwd()) - -from main.library.audio.features import change_rms -from main.app.variables import config, logger, translations, configs -from main.library.utils import load_embedders_model, extract_features, load_faiss_index, load_model - -class RealtimeVoiceConverter: - def __init__(self, weight_root, noise_scale = 0.35): - self.model = None - self.net_g = None - self.tgt_sr = None - self.use_f0 = None - self.version = None - self.energy = None - self.architecture = None - self.setup(weight_root, noise_scale) - - def setup(self, weight_root, noise_scale = 0.35): - model = load_model(weight_root) - - if weight_root.endswith(".pth"): - from main.library.algorithm.synthesizers import Synthesizer, SynthesizerSVC - - self.tgt_sr = model["config"][-1] - model["config"][-3] = model["weight"]["emb_g.weight"].shape[0] - - self.use_f0 = model.get("f0", 1) - self.version = model.get("version", "v1") - self.vocoder = model.get("vocoder", "Default") - self.energy = model.get("energy", False) - self.architecture = model.get("architecture", "RVC") - - if self.vocoder != "Default": config.is_half = False - - if self.architecture == "RVC": - net_g = Synthesizer( - *model["config"], - use_f0=self.use_f0, - text_enc_hidden_dim=768 if self.version == "v2" else 256, - vocoder=self.vocoder, - checkpointing=False, - energy=self.energy - ) - else: - net_g = SynthesizerSVC( - *model["config"], - text_enc_hidden_dim=768 if self.version == "v2" else 256, - vocoder=self.vocoder, - noise_scale=noise_scale - ) - - net_g.load_state_dict(model["weight"], strict=False) - net_g.eval().to(config.device).to(torch.float16 if config.is_half else torch.float32) - net_g.remove_weight_norm() - if config.compile_all: net_g = torch.compile(net_g, mode=config.compile_mode) - - self.net_g = net_g - self.model = model - else: - self.model = model - self.net_g = self.model.to(config.device) - self.tgt_sr = self.model.cpt.get("tgt_sr", 32000) - self.use_f0 = self.model.cpt.get("f0", 1) - self.version = self.model.cpt.get("version", "v1") - self.energy = self.model.cpt.get("energy", False) - self.architecture = self.model.cpt.get("architecture", "RVC") - - return self - - def inference( - self, - feats, - p_len, - sid, - pitch, - pitchf, - energy - ): - output = self.net_g.infer( - feats, - p_len, - pitch, - pitchf, - sid, - energy - )[0][0, 0] - - return torch.clip( - output, - -1.0, - 1.0, - out=output - ) - -class Pipeline: - def __init__( - self, - weight_root, - index_path=None, - f0_method = "rmvpe", - predictor_onnx=False, - embedders_mode="fairseq", - embedder_model="hubert_base", - noise_scale = 0.35, - sample_rate=16000, - hop_length=160, - nprobe=1, - sid = 0 - ): - self.vc = RealtimeVoiceConverter(weight_root, noise_scale) - - self.f0_method = f0_method - self.use_f0 = self.vc.use_f0 - self.tgt_sr = self.vc.tgt_sr - self.energy = self.vc.energy - self.version = self.vc.version - self.f0_min = configs.get("f0_min", 50) - self.f0_max = configs.get("f0_max", 1100) - - self.rms = self.setup_rms() - self.embedder = self.setup_embedder(embedder_model, embedders_mode) - self.predictor = self.setup_predictor(sample_rate, hop_length, predictor_onnx) - self.index, self.big_tsr = load_faiss_index(index_path.strip().strip('"').strip("\n").strip('"').strip().replace("trained", "added"), nprobe=nprobe) - - self.sid = sid - self.device = config.device - self.is_half = config.is_half - self.model_window = self.tgt_sr // 100 - self.dtype = torch.float16 if self.is_half else torch.float32 - - self.resamplers = {} - self.p_len = torch.zeros(1, device=self.device, dtype=torch.int64) - self.torch_sid = torch.tensor([sid], device=self.device, dtype=torch.int64) - - def setup_predictor(self, sample_rate, hop_length, predictor_onnx = False): - if self.use_f0: - from main.library.predictors.Generator import Generator - - predictor = Generator( - sample_rate=sample_rate, - hop_length=hop_length, - f0_min=configs.get("f0_min", 50), - f0_max=configs.get("f0_max", 1100), - alpha=0, - is_half=config.is_half, - device=config.device, - predictor_onnx=predictor_onnx, - return_tensor=True - ) - else: predictor = None - - return predictor - - def setup_embedder(self, embedder_model, embedders_mode): - embedder = load_embedders_model( - embedder_model, - embedders_mode=embedders_mode - ) - - if isinstance(embedder, torch.nn.Module): - dtype = torch.float16 if config.is_half else torch.float32 - embedder = embedder.to(config.device).to(dtype).eval() - if config.compile_all and embedders_mode != "whisper": embedder = torch.compile(embedder, mode=config.compile_mode) - - return embedder - - def setup_rms(self): - if self.energy: - from main.inference.extracting.rms import RMSEnergyExtractor - - rms = RMSEnergyExtractor( - frame_length=2048, - hop_length=160, - center=True, - pad_mode="reflect" - ).to(config.device).eval() - else: rms = None - - return rms - - def inference( - self, - audio, - pitch = None, - pitchf = None, - f0_up_key = 0, - index_rate = 0.5, - p_len = 0, - silence_front = 0, - skip_head = None, - return_length = None, - protect = 0.5, - filter_radius = 3, - rms_mix_rate = 1, - f0_autotune = False, - f0_autotune_strength = 1, - proposal_pitch = False, - proposal_pitch_threshold = 255.0, - torchgate = None, - board = None, - embedders_mix = False, - embedders_mix_layers = 9, - embedders_mix_ratio = 0.5, - block_size_16k = None, - ): - with torch.inference_mode(): - assert audio.dim() == 1, audio.dim() - formant_length = int(np.ceil(return_length * 1.0)) - shift = (block_size_16k or skip_head * self.predictor.window) // self.predictor.window - - f0_frame = (block_size_16k + 800 if block_size_16k else skip_head * self.window + 800) - if "rmvpe" in self.f0_method: f0_frame = 5120 * ((f0_frame - 1) // 5120 + 1) - 160 - - if self.use_f0: - pitch_new, pitchf_new = self.predictor.realtime_calculator( - audio[-min(f0_frame, audio.shape[0]):], - self.f0_method, - None, - None, - f0_up_key, - filter_radius, - f0_autotune, - f0_autotune_strength, - proposal_pitch, - proposal_pitch_threshold - ) - - pitch_new, pitchf_new = pitch_new.squeeze(0), pitchf_new.squeeze(0) - if shift > 0: - pitch[:-shift] = pitch[shift:].clone() - pitchf[:-shift] = pitchf[shift:].clone() - - interior_pitch = pitch_new[3:-1] if pitch_new.shape[0] > 4 else pitch_new - interior_pitchf = pitchf_new[3:-1] if pitchf_new.shape[0] > 4 else pitchf_new - - pitch[-interior_pitch.shape[0]:] = interior_pitch - pitchf[-interior_pitchf.shape[0]:] = interior_pitchf - else: pitch, pitchf = None, None - - energy = self.rms( - audio[silence_front:].to(self.device).unsqueeze(0) - ) if self.energy else None - - feats = extract_features( - self.embedder, - audio.view(1, -1), - self.version, - mix=embedders_mix, - mix_layers=embedders_mix_layers, - mix_ratio=embedders_mix_ratio - ) - - feats = torch.cat((feats, feats[:, -1:, :]), 1) - feats0 = feats.detach().clone() if protect < 0.5 and self.use_f0 else None - - if self.index is not None and self.big_tsr is not None and index_rate != 0: - try: - skip_offset = skip_head // 2 - score, ix = self.index.search(feats[0][skip_offset :], k=8) - weight = (1 / score).square() - query = (self.big_tsr[ix] * (weight / weight.sum(dim=1, keepdim=True)).unsqueeze(2)).sum(dim=1) - - feats[0][skip_offset :] = query.unsqueeze(0) * index_rate + (1.0 - index_rate) * feats[0][skip_offset :] - except AssertionError: - logger.warning(translations["index_assertion"]) - self.index = self.big_tsr = None - - feats = F.interpolate(feats.permute(0, 2, 1), scale_factor=2).permute(0, 2, 1)[:, :p_len, :] - - if self.use_f0: pitch, pitchf = pitch[-p_len:].unsqueeze(0), pitchf[-p_len:].unsqueeze(0) * (formant_length / return_length) - if self.energy: energy = energy[:p_len].unsqueeze(0) - - if feats0 is not None: - pitchff = pitchf.detach().clone() - pitchff[pitchf > 0] = 1 - pitchff[pitchf < 1] = protect - pitchff = pitchff.unsqueeze(-1) - - feats0 = F.interpolate( - feats0.permute(0, 2, 1), - scale_factor=2 - ).permute(0, 2, 1)[:, :p_len, :] - - feats = (feats * pitchff + feats0 * (1 - pitchff)).to(feats0.dtype) - - feats = feats.to(self.dtype) - pitch = pitch if self.use_f0 else None - pitchf = pitchf.to(self.dtype) if self.use_f0 else None - energy = energy.to(self.dtype) if self.energy else None - self.p_len.fill_(p_len) - - out_audio = self.vc.inference( - feats, - self.p_len, - self.torch_sid, - pitch, - pitchf, - energy - ) - - if rms_mix_rate != 1: - out_audio = change_rms( - audio[-(return_length * self.predictor.window):], - self.predictor.sample_rate, - out_audio, - self.tgt_sr, - rms_mix_rate, - device=self.device - ) - - if torchgate is not None: - out_audio = torchgate( - out_audio.unsqueeze(0) - ).squeeze(0) - - scaled_window = int(np.floor(1.0 * self.model_window)) - - if scaled_window != self.model_window: - if scaled_window not in self.resamplers: - self.resamplers[scaled_window] = tat.Resample( - orig_freq=scaled_window, - new_freq=self.model_window, - dtype=torch.float32 - ).to(self.device) - - out_audio = self.resamplers[scaled_window](out_audio[: return_length * scaled_window]) - - if board is not None: - out_audio = torch.as_tensor( - board( - out_audio.cpu().numpy(), - self.tgt_sr - ), - device=config.device - ) - - return out_audio.float() \ No newline at end of file diff --git a/main/inference/realtime/realtime.py b/main/inference/realtime/realtime.py deleted file mode 100644 index 163078bbfe1b8bd4673ff3016b96c1b70a274459..0000000000000000000000000000000000000000 --- a/main/inference/realtime/realtime.py +++ /dev/null @@ -1,630 +0,0 @@ -import os -import sys -import time -import torch - -import numpy as np -import torch.nn.functional as F -import torchaudio.transforms as tat - -sys.path.append(os.getcwd()) - -from main.app.variables import config, translations -from main.inference.realtime.pipeline import Pipeline -from main.library.utils import circular_write, check_assets - -class Realtime: - def __init__( - self, - model_path, - index_path = None, - f0_method = "rmvpe", - predictor_onnx = False, - embedder_model = "hubert_base", - embedders_mode = "fairseq", - sample_rate = 16000, - hop_length = 160, - silent_threshold = 0, - input_sample_rate = 48000, - output_sample_rate = 48000, - vad_enabled = False, - vad_sensitivity = 3, - vad_frame_ms = 30, - clean_audio=False, - clean_strength=0.7, - post_process = False, - sid = 0, - noise_scale = 0.35, - nprobe=1, - **kwargs - ): - check_assets( - f0_method, - embedder_model, - predictor_onnx=predictor_onnx, - embedders_mode=embedders_mode - ) - - self.model_path = model_path - self.index_path = index_path - self.f0_method = f0_method - self.predictor_onnx = predictor_onnx - self.embedder_model = embedder_model - self.embedders_mode = embedders_mode - self.sample_rate = sample_rate - self.hop_length = hop_length - self.post_process = post_process - self.sid = sid - self.noise_scale = noise_scale - self.nprobe = nprobe - self.kwargs = kwargs - self.pipeline = None - self.convert_buffer = None - self.pitch_buffer = None - self.pitchf_buffer = None - self.return_length = 0 - self.skip_head = 0 - self.silence_front = 0 - self.resample_in = None - self.resample_out = None - self.vad = None - self.tg = None - self.input_sample_rate = input_sample_rate - self.output_sample_rate = output_sample_rate - self.vad_enabled = vad_enabled - self.vad_sensitivity = vad_sensitivity - self.vad_frame_ms = vad_frame_ms - self.clean_audio = clean_audio - self.clean_strength = clean_strength - self.input_sensitivity = 10 ** (silent_threshold / 20) - self.window_size = sample_rate // 100 - self.dtype = torch.float16 if config.is_half else torch.float32 - - if self.vad_enabled: - from main.inference.realtime.vad_utils import VADProcessor - - self.vad = VADProcessor( - sensitivity_mode=self.vad_sensitivity, - sample_rate=self.sample_rate, - frame_duration_ms=self.vad_frame_ms - ) - else: self.vad = None - - self.board = self.setup_pedalboard(**self.kwargs) if self.post_process else None - - self.pipeline = Pipeline( - weight_root=self.model_path, - index_path=self.index_path, - f0_method=self.f0_method, - predictor_onnx=self.predictor_onnx, - embedders_mode=self.embedders_mode, - embedder_model=self.embedder_model, - noise_scale=self.noise_scale, - sample_rate=self.sample_rate, - hop_length=self.hop_length, - nprobe=self.nprobe, - sid=self.sid - ) - - if self.clean_audio: - from main.library.audio.noisereduce import TorchGate - - self.tg = TorchGate( - self.pipeline.tgt_sr, - prop_decrease=self.clean_strength - ).to(config.device) - else: self.tg = None - - self.resample_in = tat.Resample( - orig_freq=self.input_sample_rate, - new_freq=self.sample_rate, - dtype=torch.float32 - ).to(config.device) - self.resample_out = tat.Resample( - orig_freq=self.pipeline.tgt_sr, - new_freq=self.output_sample_rate, - dtype=torch.float32 - ).to(config.device) - - def setup_pedalboard(self, **kwargs): - from pedalboard import ( - Pedalboard, - Chorus, - Distortion, - Reverb, - PitchShift, - Delay, - Limiter, - Gain, - Bitcrush, - Clipping, - Compressor, - Phaser, - HighpassFilter - ) - - board = Pedalboard([HighpassFilter()]) - - if kwargs["chorus"]: - board.append( - Chorus( - depth=kwargs["chorus_depth"], - rate_hz=kwargs["chorus_rate"], - mix=kwargs["chorus_mix"], - centre_delay_ms=kwargs["chorus_delay"], - feedback=kwargs["chorus_feedback"] - ) - ) - - if kwargs["distortion"]: - board.append( - Distortion( - drive_db=kwargs["distortion_gain"] - ) - ) - - if kwargs["reverb"]: - board.append( - Reverb( - room_size=kwargs["reverb_room_size"], - damping=kwargs["reverb_damping"], - wet_level=kwargs["reverb_wet_level"], - dry_level=kwargs["reverb_dry_level"], - width=kwargs["reverb_width"], - freeze_mode=int(kwargs["reverb_freeze_mode"]) - ) - ) - - if kwargs["pitch_shift"]: - board.append( - PitchShift( - semitones=kwargs["pitch_shift_semitones"] - ) - ) - - if kwargs["delay"]: - board.append( - Delay( - delay_seconds=kwargs["delay_seconds"], - feedback=kwargs["delay_feedback"], - mix=kwargs["delay_mix"] - ) - ) - - if kwargs["compressor"]: - board.append( - Compressor( - threshold_db=kwargs["compressor_threshold"], - ratio=kwargs["compressor_ratio"], - attack_ms=kwargs["compressor_attack"], - release_ms=kwargs["compressor_release"] - ) - ) - - if kwargs["limiter"]: - board.append( - Limiter( - threshold_db=kwargs["limiter_threshold"], - release_ms=kwargs["limiter_release"] - ) - ) - - if kwargs["gain"]: - board.append( - Gain( - gain_db=kwargs["gain_db"] - ) - ) - - if kwargs["bitcrush"]: - board.append( - Bitcrush( - bit_depth=kwargs["bitcrush_bit_depth"] - ) - ) - - if kwargs["clipping"]: - board.append( - Clipping( - threshold_db=kwargs["clipping_threshold"] - ) - ) - - if kwargs["phaser"]: - board.append( - Phaser( - rate_hz=kwargs["phaser_rate_hz"], - depth=kwargs["phaser_depth"], - centre_frequency_hz=kwargs["phaser_centre_frequency_hz"], - feedback=kwargs["phaser_feedback"], - mix=kwargs["phaser_mix"] - ) - ) - - return board - - def realloc( - self, - block_frame, - extra_frame, - crossfade_frame, - sola_search_frame - ): - block_frame_16k = int( - block_frame / self.input_sample_rate * self.sample_rate - ) - crossfade_frame_16k = int( - crossfade_frame / self.input_sample_rate * self.sample_rate - ) - - sola_search_frame_16k = int( - sola_search_frame / self.input_sample_rate * self.sample_rate - ) - extra_frame_16k = int( - extra_frame / self.input_sample_rate * self.sample_rate - ) - - convert_size_16k = ( - block_frame_16k + - sola_search_frame_16k + - extra_frame_16k + - crossfade_frame_16k - ) - - if (modulo := convert_size_16k % self.window_size) != 0: - convert_size_16k = convert_size_16k + (self.window_size - modulo) - - self.convert_feature_size_16k = convert_size_16k // self.window_size - self.block_frame_16k = block_frame_16k - self.skip_head = extra_frame_16k // self.window_size - self.return_length = self.convert_feature_size_16k - self.skip_head - self.silence_front = extra_frame_16k - (self.window_size * 5) if self.silence_front else 0 - self.warmup_blocks = int(np.ceil(convert_size_16k / block_frame_16k)) + 1 - - audio_buffer_size = block_frame_16k + crossfade_frame_16k - - self.audio_buffer = torch.zeros( - audio_buffer_size, - dtype=self.dtype, - device=config.device - ) - self.convert_buffer = torch.zeros( - convert_size_16k, - dtype=self.dtype, - device=config.device - ) - self.pitch_buffer = torch.zeros( - self.convert_feature_size_16k + 1, - dtype=torch.int64, - device=config.device - ) - self.pitchf_buffer = torch.zeros( - self.convert_feature_size_16k + 1, - dtype=self.dtype, - device=config.device - ) - - def inference( - self, - audio_in, - f0_up_key = 0, - index_rate = 0.5, - protect = 0.5, - filter_radius = 3, - rms_mix_rate = 1, - f0_autotune = False, - f0_autotune_strength = 1, - proposal_pitch = False, - proposal_pitch_threshold = 255.0, - embedders_mix = False, - embedders_mix_layers = 9, - embedders_mix_ratio = 0.5 - ): - if self.pipeline is None: - raise RuntimeError(translations["create_pipeline_error"]) - - audio_in_16k = self.resample_in( - torch.as_tensor( - audio_in, - dtype=torch.float32, - device=config.device - ) - ).to(self.dtype) - - circular_write(audio_in_16k, self.audio_buffer) - - vol_t = self.audio_buffer.square().mean().sqrt() - vol = max(vol_t.item(), 0) - - tg = self.tg - board = self.board - - def inference_with_silent(): - self.pipeline.inference( - self.convert_buffer, - self.pitch_buffer, - self.pitchf_buffer, - f0_up_key, - index_rate, - self.convert_feature_size_16k, - self.silence_front, - self.skip_head, - self.return_length, - protect, - filter_radius, - rms_mix_rate, - f0_autotune, - f0_autotune_strength, - proposal_pitch, - proposal_pitch_threshold, - tg, - board, - embedders_mix, - embedders_mix_layers, - embedders_mix_ratio, - block_size_16k=self.block_frame_16k - ) - - return None, vol - - if self.warmup_blocks > 0: - self.warmup_blocks -= 1 - circular_write(audio_in_16k, self.convert_buffer) - return inference_with_silent() - - if self.vad is not None: - is_speech = self.vad.is_speech(audio_in_16k.cpu().numpy().copy()) - if not is_speech: return inference_with_silent() - - if vol < self.input_sensitivity: return inference_with_silent() - circular_write(audio_in_16k, self.convert_buffer) - - audio_model = self.pipeline.inference( - self.convert_buffer, - self.pitch_buffer, - self.pitchf_buffer, - f0_up_key, - index_rate, - self.convert_feature_size_16k, - self.silence_front, - self.skip_head, - self.return_length, - protect, - filter_radius, - rms_mix_rate, - f0_autotune, - f0_autotune_strength, - proposal_pitch, - proposal_pitch_threshold, - tg, - board, - embedders_mix, - embedders_mix_layers, - embedders_mix_ratio, - block_size_16k=self.block_frame_16k - ) - - audio_out = self.resample_out(audio_model * vol_t.sqrt()) - return audio_out, vol - -class VoiceChanger: - def __init__( - self, - read_chunk_size, - cross_fade_overlap_size, - input_sample_rate, - output_sample_rate, - extra_convert_size, - model_path, - index_path, - f0_method, - predictor_onnx, - embedder_model, - embedders_mode, - sample_rate, - hop_length, - silent_threshold, - vad_enabled, - vad_sensitivity, - vad_frame_ms, - clean_audio, - clean_strength, - post_process, - sid, - noise_scale, - nprobe, - record_audio, - record_audio_path, - export_format, - **kwargs - ): - self.input_sample_rate = input_sample_rate - self.output_sample_rate = output_sample_rate - self.block_frame = read_chunk_size * 128 - self.crossfade_frame = int(cross_fade_overlap_size * input_sample_rate) - self.extra_frame = int(extra_convert_size * input_sample_rate) - self.sola_search_frame = input_sample_rate // 100 - self.cor_den_ones = torch.ones(1, 1, self.crossfade_frame, device=config.device, dtype=torch.float32) - self.vc_model = Realtime( - model_path, - index_path, - f0_method, - predictor_onnx, - embedder_model, - embedders_mode, - sample_rate, - hop_length, - silent_threshold, - self.input_sample_rate, - self.output_sample_rate, - vad_enabled, - vad_sensitivity, - vad_frame_ms, - clean_audio, - clean_strength, - post_process, - sid, - noise_scale, - nprobe, - **kwargs - ) - self.record_audio = record_audio - self.record_audio_path = record_audio_path - self.export_format = export_format - self.sola_buffer = None - self.vc_model.realloc( - self.block_frame, - self.extra_frame, - self.crossfade_frame, - self.sola_search_frame - ) - self.generate_strength() - self.setup_soundfile_record() - - def setup_soundfile_record(self): - import soundfile as sf - - self.soundfile = sf.SoundFile( - self.record_audio_path, - mode="w", - samplerate=self.output_sample_rate, - channels=1, - format=self.export_format.lower(), - ) if self.record_audio else None - - def generate_strength(self): - self.fade_in_window = ( - 0.5 * np.pi * torch.linspace(0.0, 1.0, steps=self.crossfade_frame, device=config.device, dtype=torch.float32) - ).sin() ** 2 - self.fade_out_window = 1 - self.fade_in_window - - self.sola_buffer = torch.zeros( - self.crossfade_frame, - device=config.device, - dtype=torch.float32 - ) - - def process_audio( - self, - audio_in, - f0_up_key = 0, - index_rate = 0.5, - protect = 0.5, - filter_radius = 3, - rms_mix_rate = 1, - f0_autotune = False, - f0_autotune_strength = 1, - proposal_pitch = False, - proposal_pitch_threshold = 255.0, - embedders_mix = False, - embedders_mix_layers = 9, - embedders_mix_ratio = 0.5 - ): - block_size = audio_in.shape[0] - - audio, vol = self.vc_model.inference( - audio_in, - f0_up_key, - index_rate, - protect, - filter_radius, - rms_mix_rate, - f0_autotune, - f0_autotune_strength, - proposal_pitch, - proposal_pitch_threshold, - embedders_mix, - embedders_mix_layers, - embedders_mix_ratio - ) - - if audio is None: - self.sola_buffer.zero_() - audio = np.zeros(block_size, dtype=np.float32) - - if self.record_audio and self.soundfile is not None: self.soundfile.write(audio) - return audio, vol - - is_onset = not self.sola_buffer.any() - conv_input = audio[None, None, : self.crossfade_frame + self.sola_search_frame].float() - cor_nom = F.conv1d(conv_input, self.sola_buffer[None, None, :]) - - cor_den = ( - F.conv1d( - conv_input ** 2, - self.cor_den_ones - ) + 1e-8 - ).sqrt() - - sola_offset = (cor_nom[0, 0] / cor_den[0, 0]).argmax() - audio = audio[sola_offset:] - - if is_onset: - n_hops = block_size // 160 - if n_hops >= 1: - hop_energy = audio[: n_hops * 160].reshape(n_hops, 160).abs().max(dim=1).values - peak = hop_energy.max().item() - onset_sample = 0 - - if peak > 1e-4: - above = (hop_energy > 0.1 * peak).nonzero(as_tuple=False) - if len(above) > 0: onset_sample = int(above[0].item()) * 160 - else: onset_sample = 0 - - audio[:onset_sample] = 0.0 - fade_len = min(block_size - onset_sample, self.crossfade_frame) - if fade_len > 0: audio[onset_sample : onset_sample + fade_len] *= self.fade_in_window[:fade_len] - else: - audio[: self.crossfade_frame] *= self.fade_in_window - audio[: self.crossfade_frame] += self.sola_buffer * self.fade_out_window - - min_len = block_size + self.crossfade_frame - if audio.shape[0] < min_len: audio = torch.cat([audio, torch.zeros(min_len - audio.shape[0], device=config.device, dtype=audio.dtype)]) - - self.sola_buffer[:] = audio[block_size : block_size + self.crossfade_frame] - audio_output = audio[:block_size].detach().cpu().numpy() - - if self.record_audio and self.soundfile is not None: self.soundfile.write(audio_output) - return audio_output, vol - - @torch.no_grad() - def on_request( - self, - audio_in, - f0_up_key = 0, - index_rate = 0.5, - protect = 0.5, - filter_radius = 3, - rms_mix_rate = 1, - f0_autotune = False, - f0_autotune_strength = 1, - proposal_pitch = False, - proposal_pitch_threshold = 255.0, - embedders_mix = False, - embedders_mix_layers = 9, - embedders_mix_ratio = 0.5 - ): - if self.vc_model is None: - raise RuntimeError(translations["voice_changer_selected_error"]) - - start = time.perf_counter() - - result, vol = self.process_audio( - audio_in, - f0_up_key, - index_rate, - protect, - filter_radius, - rms_mix_rate, - f0_autotune, - f0_autotune_strength, - proposal_pitch, - proposal_pitch_threshold, - embedders_mix, - embedders_mix_layers, - embedders_mix_ratio - ) - - end = time.perf_counter() - - return result, vol, [0, (end - start) * 1000, 0] \ No newline at end of file diff --git a/main/inference/realtime/vad_utils.py b/main/inference/realtime/vad_utils.py deleted file mode 100644 index f093a45ac27cedef746e47ae99ec9498025f8521..0000000000000000000000000000000000000000 --- a/main/inference/realtime/vad_utils.py +++ /dev/null @@ -1,85 +0,0 @@ -import webrtcvad - -import numpy as np - -class VADProcessor: - def __init__( - self, - sensitivity_mode=3, - sample_rate=16000, - frame_duration_ms=30 - ): - if sample_rate not in [8000, 16000, 32000, 48000]: raise ValueError - if frame_duration_ms not in [10, 20, 30]: raise ValueError - - self.sample_rate = sample_rate - self.frame_duration_ms = frame_duration_ms - - self.vad = webrtcvad.Vad(sensitivity_mode) - self.frame_length = int(sample_rate * (frame_duration_ms / 1000.0)) - - def is_speech(self, audio_chunk): - if audio_chunk.ndim > 1 and audio_chunk.shape[1] == 1: audio_chunk = audio_chunk.flatten() - elif audio_chunk.ndim > 1: audio_chunk = np.mean(audio_chunk, axis=1) - - if np.max(np.abs(audio_chunk)) > 1.0: audio_chunk = np.clip(audio_chunk, -1.0, 1.0) - - audio_chunk = (audio_chunk * 32767).astype(np.int16) - num_frames = len(audio_chunk) // self.frame_length - - if num_frames == 0 and len(audio_chunk) > 0: - audio_chunk = np.concatenate(( - audio_chunk, - np.zeros( - self.frame_length - len(audio_chunk), - dtype=np.int16 - ) - )) - num_frames = 1 - elif num_frames == 0 and len(audio_chunk) == 0: return False - - try: - for i in range(num_frames): - start = i * self.frame_length - - if self.vad.is_speech( - audio_chunk[start:start + self.frame_length].tobytes(), - self.sample_rate - ): - return True - - return False - except Exception: - return False - - def get_speech(self, y, min_speech_duration=0.5): - duration = len(y) / self.sample_rate - is_speech_list, segments = [], [] - - for i in range(0, len(y), self.frame_length): - chunk = y[i:i + self.frame_length] - if len(chunk) < self.frame_length: chunk = np.pad(chunk, (0, self.frame_length - len(chunk))) - is_speech_list.append(self.vad.is_speech((chunk * 32767).astype(np.int16).tobytes(), self.sample_rate)) - - start_frame = None - silence_threshold_frames = int(500 / self.frame_duration_ms) - silence_counter = 0 - - for idx, is_speech in enumerate(is_speech_list): - if is_speech: - if start_frame is None: start_frame = idx - silence_counter = 0 - elif start_frame is not None: - silence_counter += 1 - - if silence_counter > silence_threshold_frames: - start_time = (start_frame * self.frame_duration_ms) / 1000.0 - end_time = ((idx - silence_counter) * self.frame_duration_ms) / 1000.0 - - if end_time - start_time >= min_speech_duration: segments.append({"start": start_time, "end": min(end_time, duration)}) - - start_frame = None - silence_counter = 0 - - if start_frame is not None: segments.append({"start": (start_frame * self.frame_duration_ms) / 1000.0, "end": duration}) - return segments \ No newline at end of file diff --git a/main/inference/separate_music.py b/main/inference/separate_music.py deleted file mode 100644 index 9e3a5c0bd12ec12928215876203d203cf4dc919e..0000000000000000000000000000000000000000 --- a/main/inference/separate_music.py +++ /dev/null @@ -1,700 +0,0 @@ -import os -import sys -import time -import argparse - -sys.path.append(os.getcwd()) - -from main.library.utils import strtobool -from main.library.audio.audio import pydub_load -from main.library.uvr5_lib.separator import Separator -from main.app.variables import config, configs, logger, translations, vr_models, demucs_models, mdx_models, karaoke_models, reverb_models, denoise_models - -def parse_arguments(): - parser = argparse.ArgumentParser() - parser.add_argument("--separate_music", action='store_true') - parser.add_argument("--input_path", type=str, required=True) - parser.add_argument("--output_dirs", type=str, default="./audios") - parser.add_argument("--export_format", type=str, default="wav") - parser.add_argument("--model_name", type=str, default="MDXNET_Main") - parser.add_argument("--karaoke_model", type=str, default="MDX-Version-1") - parser.add_argument("--reverb_model", type=str, default="MDX-Reverb") - parser.add_argument("--denoise_model", type=str, default="Normal") - parser.add_argument("--sample_rate", type=int, default=44100) - parser.add_argument("--shifts", type=int, default=2) - parser.add_argument("--batch_size", type=int, default=1) - parser.add_argument("--overlap", type=float, default=0.25) - parser.add_argument("--aggression", type=int, default=5) - parser.add_argument("--hop_length", type=int, default=1024) - parser.add_argument("--window_size", type=int, default=512) - parser.add_argument("--segments_size", type=int, default=256) - parser.add_argument("--post_process_threshold", type=float, default=0.2) - parser.add_argument("--enable_tta", type=lambda x: bool(strtobool(x)), default=False) - parser.add_argument("--enable_denoise", type=lambda x: bool(strtobool(x)), default=False) - parser.add_argument("--high_end_process", type=lambda x: bool(strtobool(x)), default=False) - parser.add_argument("--enable_post_process", type=lambda x: bool(strtobool(x)), default=False) - parser.add_argument("--separate_backing", type=lambda x: bool(strtobool(x)), default=False) - parser.add_argument("--separate_reverb", type=lambda x: bool(strtobool(x)), default=False) - - return parser.parse_args() - -def main(): - args = parse_arguments() - - ( - input_path, - output_dirs, - export_format, - model_name, - karaoke_model, - reverb_model, - denoise_model, - sample_rate, - shifts, - batch_size, - overlap, - aggression, - hop_length, - window_size, - segments_size, - post_process_threshold, - enable_tta, - enable_denoise, - high_end_process, - enable_post_process, - separate_backing, - separate_reverb - ) = ( - args.input_path, - args.output_dirs, - args.export_format, - args.model_name, - args.karaoke_model, - args.reverb_model, - args.denoise_model, - args.sample_rate, - args.shifts, - args.batch_size, - args.overlap, - args.aggression, - args.hop_length, - args.window_size, - args.segments_size, - args.post_process_threshold, - args.enable_tta, - args.enable_denoise, - args.high_end_process, - args.enable_post_process, - args.separate_backing, - args.separate_reverb - ) - - separate( - input_path, - output_dirs, - export_format, - model_name, - karaoke_model, - reverb_model, - denoise_model, - sample_rate, - shifts, - batch_size, - overlap, - aggression, - hop_length, - window_size, - segments_size, - post_process_threshold, - enable_tta, - enable_denoise, - high_end_process, - enable_post_process, - separate_backing, - separate_reverb - ) - -def separate( - input_path, - output_dirs, - export_format="wav", - model_name="MDXNET_Main", - karaoke_model="MDX-Version-1", - reverb_model="MDX-Reverb", - denoise_model="Normal", - sample_rate=44100, - shifts=2, - batch_size=1, - overlap=0.25, - aggression=5, - hop_length=1024, - window_size=512, - segments_size=256, - post_process_threshold=0.2, - enable_tta=False, - enable_denoise=False, - high_end_process=False, - enable_post_process=False, - separate_backing=False, - separate_reverb=False -): - start_time = time.time() - pid_path = os.path.join("assets", "separate_pid.txt") - - with open(pid_path, "w") as pid_file: - pid_file.write(str(os.getpid())) - - try: - input_path = input_path.strip(" ").strip('"').strip("\n").strip('"').strip(" ") - output_dirs = os.path.dirname(output_dirs) or output_dirs - - log_data = { - translations["audio_path"]: input_path, - translations["output_path"]: output_dirs, - translations["export_format"]: export_format, - translations["shift"]: shifts, - translations["segments_size"]: segments_size, - translations["overlap"]: overlap, - translations["modelname"]: model_name, - translations["denoise_mdx"]: enable_denoise, - translations["hop_length"]: hop_length, - translations["batch_size"]: batch_size, - translations["sr"]: sample_rate, - translations["separator_backing"]: separate_backing, - translations["dereveb_audio"]: separate_reverb, - translations["aggression"]: aggression, - translations["window_size"]: window_size, - translations["post_process_threshold"]: post_process_threshold, - translations["enable_tta"]: enable_tta, - translations["high_end_process"]: high_end_process, - translations["enable_post_process"]: enable_post_process, - translations["backing_model_ver"]: karaoke_model, - translations["dereveb_model"]: reverb_model, - translations["denoise_model"]: denoise_model - } - - for key, value in log_data.items(): - logger.debug(f"{key}: {value}") - - output_files = [] - - files = [ - os.path.join(input_path, f) - for f in os.listdir(input_path) - ] if os.path.isdir(input_path) else [input_path] - - for file in files: - if os.path.isfile(file): - output_files.append(_separate( - input_path, - output_dirs, - model_name, - karaoke_model, - reverb_model, - denoise_model, - export_format, - sample_rate, - shifts, - batch_size, - overlap, - aggression, - hop_length, - window_size, - segments_size, - post_process_threshold, - enable_tta, - enable_denoise, - high_end_process, - enable_post_process, - separate_backing, - separate_reverb - )) - except Exception as e: - logger.error(f"{translations['separator_error']}: {e}") - import traceback - logger.debug(traceback.format_exc()) - - if os.path.exists(pid_path): os.remove(pid_path) - elapsed_time = time.time() - start_time - - logger.info(translations["separator_success"].format(elapsed_time=f"{elapsed_time:.2f}")) - return output_files - -def _separate( - input_path, - output_dirs, - model_name, - karaoke_model="MDX-Version-1", - reverb_model="MDX-Reverb", - denoise_model="Normal", - export_format="wav", - sample_rate=44100, - shifts=2, - batch_size=1, - overlap=0.25, - aggression=5, - hop_length=1024, - window_size=512, - segments_size=256, - post_process_threshold=0.2, - enable_tta=False, - enable_denoise=False, - high_end_process=False, - enable_post_process=False, - separate_backing=False, - separate_reverb=False -): - main_vocals, backing_vocals = None, None - - filename, _ = os.path.splitext(os.path.basename(input_path)) - output_dirs = os.path.join(output_dirs, filename) - - os.makedirs(output_dirs, exist_ok=True) - clean_file(output_dirs, export_format) - - if model_name in list(demucs_models.keys()): - original_vocals, instruments = demucs_main( - input_path, - output_dirs, - model_name, - export_format, - segments_size, - overlap, - shifts, - sample_rate - ) - elif model_name in list(vr_models.keys()): - original_vocals, instruments = vr_main( - input_path, - output_dirs, - vr_models.get(model_name, model_name), - export_format, - batch_size, - window_size, - aggression, - enable_denoise, - denoise_model, - enable_tta, - enable_post_process, - post_process_threshold, - high_end_process, - sample_rate, - ) - else: - original_vocals, instruments = mdx_main( - input_path, - output_dirs, - mdx_models.get(model_name, model_name), - export_format, - segments_size, - overlap, - enable_denoise, - hop_length, - batch_size, - sample_rate, - ) - - if separate_backing: - if karaoke_model.startswith("MDX"): - main_vocals, backing_vocals = mdx_main( - original_vocals, - output_dirs, - karaoke_models.get(karaoke_model, karaoke_model), - export_format, - segments_size, - overlap, - enable_denoise, - hop_length, - batch_size, - sample_rate, - mode="karaoke" - ) - else: - main_vocals, backing_vocals = vr_main( - original_vocals, - output_dirs, - karaoke_models.get(karaoke_model, karaoke_model), - export_format, - batch_size, - window_size, - aggression, - enable_denoise, - denoise_model, - enable_tta, - enable_post_process, - post_process_threshold, - high_end_process, - sample_rate, - mode="karaoke" - ) - - if separate_reverb: - dereverb = [original_vocals] - if separate_backing: dereverb.append(main_vocals) - - for audio in dereverb: - if karaoke_model.startswith("MDX"): - _, no_reverb_vocals = mdx_main( - audio, - output_dirs, - reverb_models.get(reverb_model, reverb_model), - export_format, - segments_size, - overlap, - enable_denoise, - hop_length, - batch_size, - sample_rate, - mode="reverb" - ) - else: - _, no_reverb_vocals = vr_main( - audio, - output_dirs, - reverb_models.get(reverb_model, reverb_model), - export_format, - batch_size, - window_size, - aggression, - enable_denoise, - denoise_model, - enable_tta, - enable_post_process, - post_process_threshold, - high_end_process, - sample_rate, - mode="reverb" - ) - - if "Original_Vocals" in os.path.basename(no_reverb_vocals): original_vocals = no_reverb_vocals - else: main_vocals = no_reverb_vocals - - return original_vocals, instruments, main_vocals, backing_vocals - -def vr_main( - input_path, - output_dirs, - model_name, - export_format="wav", - batch_size=1, - window_size=512, - aggression=5, - enable_denoise=False, - denoise_model="Normal", - enable_tta=False, - enable_post_process=False, - post_process_threshold=0.2, - high_end_process=False, - sample_rate=44100, - mode="original" -): - exists_file(input_path, output_dirs) - - logger.info(f"{translations['separator_process_2']}...") - - output_list = separate_main( - audio_file=input_path, - model_filename=model_name, - export_format=export_format, - output_dir=output_dirs, - batch_size=batch_size, - window_size=window_size, - aggression=aggression, - enable_tta=enable_tta, - enable_post_process=enable_post_process, - post_process_threshold=post_process_threshold, - high_end_process=high_end_process, - sample_rate=sample_rate - ) - - if enable_denoise: - denoise_list = [] - for audio in output_list: - audio_path = os.path.join(output_dirs, audio) - - denoise_file = separate_main( - audio_file=audio_path, - model_filename=denoise_models.get(denoise_model, denoise_model), - export_format=export_format, - output_dir=output_dirs, - batch_size=batch_size, - window_size=window_size, - aggression=aggression, - enable_tta=enable_tta, - enable_post_process=enable_post_process, - post_process_threshold=post_process_threshold, - high_end_process=high_end_process, - sample_rate=sample_rate - ) - - if os.path.exists(audio_path): os.remove(audio_path) - - for file in denoise_file: - file_path = os.path.join(output_dirs, file) - - if "_(Noise)_" in file and os.path.exists(file_path): os.remove(file_path) - elif "_(No Noise)_" in file: - filename = "".join([file.split("_(No Noise)_")[0], ".", export_format]) - os.rename(file_path, os.path.join(output_dirs, filename)) - - denoise_list.append(filename) - - logger.info(translations["separator_success_2"]) - - return process_file( - denoise_list if enable_denoise else output_list, - output_dirs, - export_format, - mode - ) - -def demucs_main( - input_path, - output_dirs, - model_name, - export_format="wav", - segments_size=256, - overlap=0.25, - shifts=2, - sample_rate=44100 -): - exists_file(input_path, output_dirs) - logger.info(f"{translations['separator_process_2']}...") - - output_list = separate_main( - audio_file=input_path, - output_dir=output_dirs, - model_filename=demucs_models.get(model_name, model_name), - export_format=export_format, - segment_size=int(segments_size // 4), - overlap=overlap, - shifts=shifts, - sample_rate=sample_rate - ) - - logger.info(translations["separator_success_2"]) - - return process_file( - output_list, - output_dirs, - export_format, - mode="4stem" - ) - -def mdx_main( - input_path, - output_dirs, - model_name, - export_format="wav", - segments_size=256, - overlap=0.25, - enable_denoise=False, - hop_length=1024, - batch_size=1, - sample_rate=44100, - mode="original" -): - exists_file(input_path, output_dirs) - logger.info(f"{translations['separator_process_2']}...") - - output_list = separate_main( - audio_file=input_path, - model_filename=model_name, - export_format=export_format, - output_dir=output_dirs, - segment_size=segments_size, - overlap=overlap, - batch_size=batch_size, - hop_length=hop_length, - enable_denoise=enable_denoise, - sample_rate=sample_rate - ) - - logger.info(translations["separator_success_2"]) - - return process_file( - output_list, - output_dirs, - export_format, - mode - ) - -def process_file( - input_list, - output_dirs, - export_format="wav", - mode="original" -): - demucs_inst = [] - reverb_audio, no_reverb_audio = None, None - - main_audio, backing_audio = ( - os.path.join(output_dirs, f"Main_Vocals.{export_format}"), - os.path.join(output_dirs, f"Backing_Vocals.{export_format}") - ) - - original_audio, instruments_audio = ( - os.path.join(output_dirs, f"Original_Vocals.{export_format}"), - os.path.join(output_dirs, f"Instruments.{export_format}") - ) - - for file in input_list: - file_path = os.path.join(output_dirs, file) - if not os.path.exists(file_path): - logger.warning(translations["not_found"].format(name=file_path)) - - if mode == "original": - if "_(Instrumental)_" in file: - os.rename(file_path, instruments_audio) - elif "_(Vocals)_" in file: - os.rename(file_path, original_audio) - elif mode == "4stem": - if "_(Vocals)_" in file: - os.rename(file_path, original_audio) - elif "_(Drums)_" in file or "_(Bass)_" in file or "_(Other)_" in file or "_(Guitar)_" in file or "_(Piano)_" in file: - demucs_inst.append(file_path) - elif mode == "reverb": - filename = file.split("_(")[0] - - reverb_audio = os.path.join( - output_dirs, - "".join([filename, "_Reverb.", export_format]) - ) - - no_reverb_audio = os.path.join( - output_dirs, - "".join([filename, "_No_Reverb.", export_format]) - ) - - if "_(Reverb)_" in file or "_(Echo)_" in file: - os.rename(file_path, reverb_audio) - elif "_(No Reverb)_" in file or "_(No Echo)_" in file: - os.rename(file_path, no_reverb_audio) - elif mode == "karaoke": - if "_(Instrumental)_" in file: - os.rename(file_path, backing_audio) - elif "_(Vocals)_" in file: - os.rename(file_path, main_audio) - - if mode == "reverb": return reverb_audio, no_reverb_audio - if mode == "karaoke": return main_audio, backing_audio - - if mode == "4stem": - demucs_audio = pydub_load(demucs_inst[0]) - for file in demucs_inst[1:]: - demucs_audio = demucs_audio.overlay(pydub_load(file)) - - demucs_audio.export(instruments_audio, format=export_format) - - if configs.get("del4stem", True): - for f in demucs_inst: - if os.path.exists(f): os.remove(f) - - return original_audio, instruments_audio - -def exists_file(input_path, output_dirs): - if not os.path.exists(input_path): - logger.warning(translations["input_not_valid"]) - sys.exit(1) - - if not os.path.exists(output_dirs): - logger.warning(translations["output_not_valid"]) - sys.exit(1) - -def clean_file(output_dirs, export_format): - for f in os.listdir(output_dirs): - if f.endswith(export_format): - file_path = os.path.join(output_dirs, f) - if os.path.exists(file_path): os.remove(file_path) - -def separate_main( - audio_file=None, - model_filename="UVR-MDX-NET_Main_340.onnx", - export_format="wav", - output_dir=".", - segment_size=256, - overlap=0.25, - batch_size=1, - hop_length=1024, - enable_denoise=False, - shifts=2, - window_size=512, - aggression=5, - enable_tta=False, - enable_post_process=False, - post_process_threshold=0.2, - high_end_process=False, - sample_rate=44100 -): - try: - separator = Separator( - logger=logger, - output_dir=output_dir, - output_format=export_format, - output_bitrate=None, - normalization_threshold=0.9, - sample_rate=sample_rate, - mdx_params={ - "hop_length": hop_length, - "segment_size": segment_size, - "overlap": overlap, - "batch_size": batch_size, - "enable_denoise": enable_denoise - }, - demucs_params={ - "segment_size": segment_size, - "shifts": shifts, - "overlap": overlap, - "segments_enabled": config.configs.get("demucs_segments_enable", True) - }, - vr_params={ - "batch_size": batch_size, - "window_size": window_size, - "aggression": aggression, - "enable_tta": enable_tta, - "enable_post_process": enable_post_process, - "post_process_threshold": post_process_threshold, - "high_end_process": high_end_process - } - ) - separator.load_model(model_filename=model_filename) - - return separator.separate(audio_file) - except: - logger.debug(translations["default_setting"]) - - separator = Separator( - logger=logger, - output_dir=output_dir, - output_format=export_format, - output_bitrate=None, - normalization_threshold=0.9, - sample_rate=44100, - mdx_params={ - "hop_length": 1024, - "segment_size": 256, - "overlap": 0.25, - "batch_size": 1, - "enable_denoise": enable_denoise - }, - demucs_params={ - "segment_size": 128, - "shifts": 2, - "overlap": 0.25, - "segments_enabled": config.configs.get("demucs_segments_enable", True) - }, - vr_params={ - "batch_size": 1, - "window_size": 512, - "aggression": 5, - "enable_tta": False, - "enable_post_process": False, - "post_process_threshold": 0.2, - "high_end_process": False - } - ) - separator.load_model(model_filename=model_filename) - - return separator.separate(audio_file) - -if __name__ == "__main__": main() \ No newline at end of file diff --git a/main/inference/training/data_utils.py b/main/inference/training/data_utils.py deleted file mode 100644 index aafea71d2d45b08f411d5feb700ddf505d965ad7..0000000000000000000000000000000000000000 --- a/main/inference/training/data_utils.py +++ /dev/null @@ -1,296 +0,0 @@ -import os -import sys -import torch - -import numpy as np -import torch.utils.data as tdata - -sys.path.append(os.getcwd()) - -from main.app.variables import translations -from main.inference.training.mel_processing import spectrogram_torch -from main.inference.training.utils import load_filepaths_and_text, load_wav_to_torch - -class TextAudioLoader(tdata.Dataset): - def __init__( - self, - hparams, - spec_dirs=None, - cache_spectrogram=True, - pitch_guidance=True, - energy=False - ): - self.audiopaths_and_text = load_filepaths_and_text(hparams.training_files) - self.max_wav_value = hparams.max_wav_value - self.filter_length = hparams.filter_length - self.sample_rate = hparams.sample_rate - self.hop_length = hparams.hop_length - self.win_length = hparams.win_length - self.max_text_len = getattr(hparams, "max_text_len", 5000) - self.min_text_len = getattr(hparams, "min_text_len", 1) - self.cache_spectrogram = cache_spectrogram - self.pitch_guidance = pitch_guidance - self.spec_dirs = spec_dirs - self.energy = energy - self._filter() - - def _filter(self): - audiopaths_and_text_new, lengths = [], [] - - for item in self.audiopaths_and_text: - audiopath = item[0] - text = item[1] - - if self.min_text_len <= len(text) and len(text) <= self.max_text_len: - audiopaths_and_text_new.append(item) - lengths.append(os.path.getsize(audiopath) // (3 * self.hop_length)) - - self.audiopaths_and_text = audiopaths_and_text_new - self.lengths = lengths - - def get_sid(self, sid): - try: - sid = torch.LongTensor([int(sid)]) - except ValueError: - sid = torch.LongTensor([0]) - - return sid - - def get_audio_text_pair(self, audiopath_and_text): - extra = audiopath_and_text[2:] - pitch, pitchf, energy, sid = None, None, None, None - - if self.pitch_guidance and self.energy: pitch, pitchf, energy, sid = extra - elif self.pitch_guidance: pitch, pitchf, sid = extra - elif self.energy: energy, sid = extra - else: sid = extra[0] - - spec, wav = self.get_audio(audiopath_and_text[0]) - dv = self.get_sid(sid) - - phone, pitch, pitchf, energy = self.get_labels( - audiopath_and_text[1], - pitch=pitch, - pitchf=pitchf, - energy=energy - ) - - len_phone = phone.size()[0] - len_spec = spec.size()[-1] - - if len_phone != len_spec: - len_min = min(len_phone, len_spec) - len_wav = len_min * self.hop_length - - spec, wav, phone = spec[:, :len_min], wav[:, :len_wav], phone[:len_min, :] - - if self.pitch_guidance: pitch, pitchf = pitch[:len_min], pitchf[:len_min] - if self.energy: energy = energy[:len_min] - - outputs = [spec, wav, phone, dv] - if self.pitch_guidance: outputs[3:3] = [pitch, pitchf] - if self.energy: outputs.append(energy) - - return tuple(outputs) - - def get_labels(self, phone, pitch=None, pitchf=None, energy=None): - phone = np.repeat(np.load(phone), 2, axis=0) - n_num = min(phone.shape[0], 900) - - return ( - torch.FloatTensor(phone[:n_num, :]), - torch.LongTensor(np.load(pitch)[:n_num]) if pitch else None, - torch.FloatTensor(np.load(pitchf)[:n_num]) if pitchf else None, - torch.FloatTensor(np.load(energy)[:n_num]) if energy else None - ) - - def get_audio(self, filename): - audio, sample_rate = load_wav_to_torch(filename) - if sample_rate != self.sample_rate: - raise ValueError(translations["sr_does_not_match"].format(sample_rate=sample_rate, sample_rate2=self.sample_rate)) - - audio_norm = audio.unsqueeze(0) - spec_filename = filename.replace(".wav", ".spec.pt") - if self.spec_dirs: spec_filename = os.path.join(self.spec_dirs, os.path.basename(spec_filename)) - - def get_spectrogram(audio_norm): - return spectrogram_torch( - audio_norm, - self.filter_length, - self.hop_length, - self.win_length, - center=False - ).squeeze(0) - - if not self.cache_spectrogram: - spec = get_spectrogram(audio_norm) - elif os.path.exists(spec_filename): - try: - spec = torch.load(spec_filename, weights_only=True) - except Exception: - spec = get_spectrogram(audio_norm) - torch.save(spec, spec_filename, _use_new_zipfile_serialization=False) - else: - spec = get_spectrogram(audio_norm) - torch.save(spec, spec_filename, _use_new_zipfile_serialization=False) - - return spec, audio_norm - - def __getitem__(self, index): - return self.get_audio_text_pair(self.audiopaths_and_text[index]) - - def __len__(self): - return len(self.audiopaths_and_text) - -class TextAudioCollate: - def __init__(self, return_ids=False, pitch_guidance=True, energy=False): - self.return_ids = return_ids - self.pitch_guidance = pitch_guidance - self.energy = energy - - def __call__(self, batch): - _, ids_sorted_decreasing = torch.sort(torch.LongTensor([x[0].size(1) for x in batch]), dim=0, descending=True) - spec_lengths, wave_lengths = torch.LongTensor(len(batch)), torch.LongTensor(len(batch)) - spec_padded, wave_padded = torch.FloatTensor(len(batch), batch[0][0].size(0), max([x[0].size(1) for x in batch])), torch.FloatTensor(len(batch), 1, max([x[1].size(1) for x in batch])) - spec_padded.zero_() - wave_padded.zero_() - - max_phone_len = max([x[2].size(0) for x in batch]) - phone_lengths, phone_padded = torch.LongTensor(len(batch)), torch.FloatTensor(len(batch), max_phone_len, batch[0][2].shape[1]) - phone_padded.zero_() - - if self.pitch_guidance: - pitch_padded, pitchf_padded = torch.LongTensor(len(batch), max_phone_len), torch.FloatTensor(len(batch), max_phone_len) - pitch_padded.zero_() - pitchf_padded.zero_() - - sid = torch.LongTensor(len(batch)) - - if self.energy: - energy_padded = torch.FloatTensor(len(batch), max_phone_len) - energy_padded.zero_() - - for i in range(len(ids_sorted_decreasing)): - row = batch[ids_sorted_decreasing[i]] - spec = row[0] - - spec_padded[i, :, : spec.size(1)] = spec - spec_lengths[i] = spec.size(1) - - wave = row[1] - wave_padded[i, :, : wave.size(1)] = wave - wave_lengths[i] = wave.size(1) - - phone = row[2] - phone_padded[i, : phone.size(0), :] = phone - phone_lengths[i] = phone.size(0) - - if self.pitch_guidance: - pitch = row[3] - pitch_padded[i, : pitch.size(0)] = pitch - pitchf = row[4] - pitchf_padded[i, : pitchf.size(0)] = pitchf - - sid[i] = row[5 if self.pitch_guidance else 3] - - if self.energy: - energy = row[6 if self.pitch_guidance else 4] - energy_padded[i, : energy.size(0)] = energy - - outputs = [phone_padded, phone_lengths, spec_padded, spec_lengths, wave_padded, wave_lengths, sid] - if self.pitch_guidance: outputs[2:2] = [pitch_padded, pitchf_padded] - if self.energy: outputs.append(energy_padded) - - return tuple(outputs) - -class DistributedBucketSampler(tdata.distributed.DistributedSampler): - def __init__( - self, - dataset, - batch_size, - boundaries, - num_replicas=None, - rank=None, - shuffle=True - ): - super().__init__( - dataset, - num_replicas=num_replicas, - rank=rank, - shuffle=shuffle - ) - self.lengths = dataset.lengths - self.batch_size = batch_size - self.boundaries = boundaries - self.buckets, self.num_samples_per_bucket = self._create_buckets() - self.total_size = sum(self.num_samples_per_bucket) - self.num_samples = self.total_size // self.num_replicas - - def _create_buckets(self): - buckets = [[] for _ in range(len(self.boundaries) - 1)] - - for i in range(len(self.lengths)): - idx_bucket = self._bisect(self.lengths[i]) - if idx_bucket != -1: buckets[idx_bucket].append(i) - - for i in range(len(buckets) - 1, -1, -1): - if len(buckets[i]) == 0: - buckets.pop(i) - self.boundaries.pop(i + 1) - - num_samples_per_bucket = [] - - for i in range(len(buckets)): - len_bucket = len(buckets[i]) - total_batch_size = self.num_replicas * self.batch_size - - num_samples_per_bucket.append( - len_bucket + ((total_batch_size - (len_bucket % total_batch_size)) % total_batch_size) - ) - - return buckets, num_samples_per_bucket - - def __iter__(self): - g = torch.Generator() - g.manual_seed(self.epoch) - indices, batches = [], [] - - if self.shuffle: - for bucket in self.buckets: - indices.append(torch.randperm(len(bucket), generator=g).tolist()) - else: - for bucket in self.buckets: - indices.append(list(range(len(bucket)))) - - for i in range(len(self.buckets)): - bucket = self.buckets[i] - len_bucket = len(bucket) - ids_bucket = indices[i] - - rem = self.num_samples_per_bucket[i] - len_bucket - ids_bucket = ( - ids_bucket + ids_bucket * (rem // len_bucket) + ids_bucket[: (rem % len_bucket)] - )[self.rank :: self.num_replicas] - - for j in range(len(ids_bucket) // self.batch_size): - batches.append([bucket[idx] for idx in ids_bucket[j * self.batch_size : (j + 1) * self.batch_size]]) - - if self.shuffle: batches = [batches[i] for i in torch.randperm(len(batches), generator=g).tolist()] - self.batches = batches - assert len(self.batches) * self.batch_size == self.num_samples - - return iter(self.batches) - - def _bisect(self, x, lo=0, hi=None): - if hi is None: hi = len(self.boundaries) - 1 - - if hi > lo: - mid = (hi + lo) // 2 - - if self.boundaries[mid] < x and x <= self.boundaries[mid + 1]: return mid - elif x <= self.boundaries[mid]: return self._bisect(x, lo, mid) - else: return self._bisect(x, mid + 1, hi) - else: return -1 - - def __len__(self): - return self.num_samples // self.batch_size \ No newline at end of file diff --git a/main/inference/training/extract_model.py b/main/inference/training/extract_model.py deleted file mode 100644 index 150867516626fd4a2842cbd1587eefc9aa0e3754..0000000000000000000000000000000000000000 --- a/main/inference/training/extract_model.py +++ /dev/null @@ -1,90 +0,0 @@ -import os -import sys -import torch -import hashlib -import datetime - -from collections import OrderedDict - -sys.path.append(os.getcwd()) - -from main.app.variables import logger, translations, config -from main.inference.training.utils import replace_keys_in_dict - -def extract_model( - ckpt, - sr, - pitch_guidance, - name, - model_path, - epoch, - step, - version, - hps, - model_author, - vocoder, - energy_use, - speakers_id, - architecture -): - try: - logger.info(translations["savemodel"].format(model_dir=model_path, epoch=epoch, step=step)) - os.makedirs(os.path.dirname(model_path), exist_ok=True) - - opt = OrderedDict( - weight={ - key: ( - value if not config.device.startswith("privateuseone") else value.detach().cpu() - ).to(torch.float16 if config.is_half else torch.float32) - for key, value in ckpt.items() - if "enc_q" not in key - } - ) - - opt["config"] = [ - hps.data.filter_length // 2 + 1, - hps.train.segment_size // hps.data.hop_length, - hps.model.inter_channels, - hps.model.hidden_channels, - hps.model.filter_channels, - hps.model.n_heads, - hps.model.n_layers, - hps.model.kernel_size, - hps.model.p_dropout, - hps.model.resblock, - hps.model.resblock_kernel_sizes, - hps.model.resblock_dilation_sizes, - hps.model.upsample_rates, - hps.model.upsample_initial_channel, - hps.model.upsample_kernel_sizes, - hps.model.spk_embed_dim, - hps.model.gin_channels, - hps.data.sample_rate - ] - opt["epoch"] = f"{epoch}epoch" - opt["step"] = step - opt["sr"] = sr - opt["f0"] = int(pitch_guidance) - opt["version"] = version - opt["creation_date"] = datetime.datetime.now().isoformat() - opt["model_hash"] = hashlib.sha256(f"{str(ckpt)} {epoch} {step} {datetime.datetime.now().isoformat()}".encode()).hexdigest() - opt["model_name"] = name - opt["author"] = model_author - opt["vocoder"] = vocoder - opt["energy"] = energy_use - opt["speakers_id"] = speakers_id - opt["architecture"] = architecture - - torch.save( - replace_keys_in_dict( - replace_keys_in_dict( - opt, - ".parametrizations.weight.original1", - ".weight_v" - ), - ".parametrizations.weight.original0", ".weight_g" - ), - model_path - ) - except Exception as e: - logger.error(f"{translations['extract_model_error']}: {e}") \ No newline at end of file diff --git a/main/inference/training/losses.py b/main/inference/training/losses.py deleted file mode 100644 index 4fa9510dd8cdd71d019580f39e75e668211d895d..0000000000000000000000000000000000000000 --- a/main/inference/training/losses.py +++ /dev/null @@ -1,51 +0,0 @@ -def feature_loss(fmap_r, fmap_g): - loss = 0 - - for dr, dg in zip(fmap_r, fmap_g): - for rl, gl in zip(dr, dg): - loss += ( - rl.float().detach() - gl.float() - ).abs().mean() - - return loss * 2 - -def discriminator_loss(disc_real_outputs, disc_generated_outputs): - loss = 0 - r_losses, g_losses = [], [] - - for dr, dg in zip(disc_real_outputs, disc_generated_outputs): - dr = dr.float() - dg = dg.float() - - r_loss = ((1 - dr) ** 2).mean() - g_loss = (dg**2).mean() - - loss += r_loss + g_loss - - r_losses.append(r_loss.item()) - g_losses.append(g_loss.item()) - - return loss, r_losses, g_losses - -def generator_loss(disc_outputs): - loss = 0 - gen_losses = [] - - for dg in disc_outputs: - l = ((1 - dg.float()) ** 2).mean() - gen_losses.append(l) - loss += l - - return loss, gen_losses - -def kl_loss(z_p, logs_q, m_p, logs_p, z_mask): - z_p = z_p.float() - logs_q = logs_q.float() - m_p = m_p.float() - logs_p = logs_p.float() - z_mask = z_mask.float() - - kl = logs_p - logs_q - 0.5 - kl += 0.5 * ((z_p - m_p) ** 2) * (-2.0 * logs_p).exp() - - return (kl * z_mask).sum() / z_mask.sum() \ No newline at end of file diff --git a/main/inference/training/mel_processing.py b/main/inference/training/mel_processing.py deleted file mode 100644 index b5f476f760c8e5f69ce5e0fe31304ffa8e4962f6..0000000000000000000000000000000000000000 --- a/main/inference/training/mel_processing.py +++ /dev/null @@ -1,190 +0,0 @@ -import os -import sys -import torch - -sys.path.append(os.getcwd()) - -from main.library.audio.features import mel - -def spectral_normalize_torch(magnitudes): - return (magnitudes.clamp(min=1e-5) * 1.0).log() - -def spectral_de_normalize_torch(magnitudes): - return magnitudes.exp() / 1.0 - -stft = None -mel_basis, hann_window = {}, {} - -def spectrogram_torch( - y, - n_fft, - hop_size, - win_size, - center=False -): - global hann_window, stft - - wnsize_dtype_device = str(win_size) + "_" + str(y.dtype) + "_" + str(y.device) - if wnsize_dtype_device not in hann_window: - hann_window[wnsize_dtype_device] = torch.hann_window(win_size).to(dtype=y.dtype, device=y.device) - - pad = torch.nn.functional.pad( - y.unsqueeze(1), - (int((n_fft - hop_size) / 2), int((n_fft - hop_size) / 2)), - mode="reflect" - ).squeeze(1) - - if str(y.device).startswith(("ocl", "privateuseone")): - if stft is None: - from main.library.backends.utils import STFT - - stft = STFT( - filter_length=n_fft, - hop_length=hop_size, - win_length=n_fft - ).to(y.device) - - spec = stft.transform( - pad.to(y.device), - eps=1e-6, - center=center - ) - else: - spec = torch.stft( - pad, - n_fft, - hop_length=hop_size, - win_length=win_size, - window=hann_window[wnsize_dtype_device].to(pad.device), - center=center, - pad_mode="reflect", - normalized=False, - onesided=True, - return_complex=True - ) - - spec = spec.abs().clamp_min_(1e-6) - - return spec.to(y.device) - -def spec_to_mel_torch( - spec, - n_fft, - num_mels, - sample_rate, - fmin, - fmax -): - global mel_basis - - fmax_dtype_device = str(fmax) + "_" + str(spec.dtype) + "_" + str(spec.device) - if fmax_dtype_device not in mel_basis: - mel_basis[fmax_dtype_device] = mel( - sr=sample_rate, - n_fft=n_fft, - n_mels=num_mels, - fmin=fmin, - fmax=fmax, - dtype=spec.dtype, - device=spec.device - ) - - return spectral_normalize_torch(mel_basis[fmax_dtype_device] @ spec) - -def mel_spectrogram_torch( - y, - n_fft, - num_mels, - sample_rate, - hop_size, - win_size, - fmin, - fmax, - center=False -): - return spec_to_mel_torch( - spectrogram_torch( - y, - n_fft, - hop_size, - win_size, - center - ), - n_fft, - num_mels, - sample_rate, - fmin, - fmax - ) - -class MultiScaleMelSpectrogramLoss(torch.nn.Module): - def __init__( - self, - sample_rate = 24000, - n_mels=[5, 10, 20, 40, 80, 160, 320], - window_lengths=[32, 64, 128, 256, 512, 1024, 2048], - loss_fn=torch.nn.L1Loss() - ): - super().__init__() - self.sample_rate = sample_rate - self.loss_fn = loss_fn - self.log_base = torch.tensor(10.0).log() - self.stft_params = [] - self.hann_window = {} - self.mel_banks = {} - self.stft_params = [(mel, win) for mel, win in zip(n_mels, window_lengths)] - - def mel_spectrogram(self, wav, n_mels, window_length): - dtype_device = str(wav.dtype) + "_" + str(wav.device) - win_dtype_device = str(window_length) + "_" + dtype_device - mel_dtype_device = str(n_mels) + "_" + dtype_device - - if win_dtype_device not in self.hann_window: - self.hann_window[win_dtype_device] = torch.hann_window(window_length, device=wav.device, dtype=torch.float32) - - wav = wav.float().squeeze(1) - - if str(wav.device).startswith(("ocl", "privateuseone")): - stft = torch.stft( - wav.cpu(), - n_fft=window_length, - hop_length=window_length // 4, - window=self.hann_window[win_dtype_device].cpu(), - return_complex=True - ) - - magnitude = stft.abs().clamp_min_(1e-6).to(wav.device, dtype=torch.float32) - else: - stft = torch.stft( - wav, - n_fft=window_length, - hop_length=window_length // 4, - window=self.hann_window[win_dtype_device], - return_complex=True - ) - - magnitude = stft.abs().clamp_min_(1e-6) - - if mel_dtype_device not in self.mel_banks: - self.mel_banks[mel_dtype_device] = mel( - sr=self.sample_rate, - n_mels=n_mels, - n_fft=window_length, - fmin=0, - fmax=None, - device=wav.device, - dtype=torch.float32 - ) - - return self.mel_banks[mel_dtype_device] @ magnitude - - def forward(self, real, fake): - loss = 0.0 - - for p in self.stft_params: - loss += self.loss_fn( - self.mel_spectrogram(real, *p).clamp(min=1e-5).log() / self.log_base, - self.mel_spectrogram(fake, *p).clamp(min=1e-5).log() / self.log_base - ) - - return loss \ No newline at end of file diff --git a/main/inference/training/train.py b/main/inference/training/train.py deleted file mode 100644 index cf7628139e6db9f0d45df323125ddedecc8b86ff..0000000000000000000000000000000000000000 --- a/main/inference/training/train.py +++ /dev/null @@ -1,1341 +0,0 @@ -import os -import re -import sys -import glob -import json -import torch -import logging -import argparse -import datetime -import warnings - -import torch.distributed as dist -import torch.multiprocessing as mp - -from tqdm import tqdm -from collections import deque -from contextlib import nullcontext -from random import randint, shuffle -from torch.utils.data import DataLoader -from torch.amp import GradScaler, autocast -from torch.utils.tensorboard import SummaryWriter - -from time import time as ttime -from torch.nn.parallel import DistributedDataParallel as DDP - -sys.path.append(os.getcwd()) -os.environ["USE_LIBUV"] = "0" if sys.platform == "win32" else "1" - -from main.library.backends import opencl, xpu -from main.app.variables import logger, translations -from main.library.utils import clear_gpu_cache, strtobool - -from main.library.algorithm import commons -from main.inference.training import losses - -from main.inference.training.extract_model import extract_model - -from main.inference.training.mel_processing import ( - MultiScaleMelSpectrogramLoss, - mel_spectrogram_torch, - spec_to_mel_torch -) - -from main.inference.training.utils import ( - HParams, - summarize, - load_checkpoint, - save_checkpoint, - load_wav_to_torch, - latest_checkpoint_path, - plot_spectrogram_to_numpy -) - -from main.app.variables import config as main_config -from main.app.variables import configs as main_configs - -if not main_config.debug_mode: - warnings.filterwarnings("ignore") - logging.getLogger("torch").setLevel(logging.ERROR) - -def parse_arguments(): - parser = argparse.ArgumentParser() - parser.add_argument("--train", action='store_true') - parser.add_argument("--model_name", type=str, required=True) - parser.add_argument("--rvc_version", type=str, default="v2") - parser.add_argument("--save_every_epoch", type=int, required=True) - parser.add_argument("--save_only_latest", type=lambda x: bool(strtobool(x)), default=True) - parser.add_argument("--save_every_weights", type=lambda x: bool(strtobool(x)), default=True) - parser.add_argument("--total_epoch", type=int, default=300) - parser.add_argument("--batch_size", type=int, default=8) - parser.add_argument("--gpu", type=str, default="0") - parser.add_argument("--pitch_guidance", type=lambda x: bool(strtobool(x)), default=True) - parser.add_argument("--g_pretrained_path", type=str, default="") - parser.add_argument("--d_pretrained_path", type=str, default="") - parser.add_argument("--overtraining_detector", type=lambda x: bool(strtobool(x)), default=False) - parser.add_argument("--overtraining_threshold", type=int, default=50) - parser.add_argument("--cleanup", type=lambda x: bool(strtobool(x)), default=False) - parser.add_argument("--cache_data_in_gpu", type=lambda x: bool(strtobool(x)), default=False) - parser.add_argument("--model_author", type=str) - parser.add_argument("--vocoder", type=str, default="Default") - parser.add_argument("--checkpointing", type=lambda x: bool(strtobool(x)), default=False) - parser.add_argument("--deterministic", type=lambda x: bool(strtobool(x)), default=False) - parser.add_argument("--benchmark", type=lambda x: bool(strtobool(x)), default=False) - parser.add_argument("--optimizer", type=str, default="AdamW") - parser.add_argument("--energy_use", type=lambda x: bool(strtobool(x)), default=False) - parser.add_argument("--use_custom_reference", type=lambda x: bool(strtobool(x)), default=False) - parser.add_argument("--reference_path", type=str, default="") - parser.add_argument("--multiscale_mel_loss", type=lambda x: bool(strtobool(x)), default=False) - parser.add_argument("--use_cosine_annealing_lr", type=lambda x: bool(strtobool(x)), default=False) - parser.add_argument("--architecture", type=str, default="RVC") - - return parser.parse_args() - -args = parse_arguments() - -( - model_name, - save_every_epoch, - total_epoch, - pretrainG, - pretrainD, - version, - gpus, - batch_size, - pitch_guidance, - save_only_latest, - save_every_weights, - cache_data_in_gpu, - overtraining_detector, - overtraining_threshold, - cleanup, - model_author, - vocoder, - checkpointing, - optimizer_choice, - energy_use, - use_custom_reference, - reference_path, - multiscale_mel_loss, - use_cosine_annealing_lr, - architecture -) = ( - args.model_name, - args.save_every_epoch, - args.total_epoch, - args.g_pretrained_path, - args.d_pretrained_path, - args.rvc_version, - args.gpu, - args.batch_size, - args.pitch_guidance, - args.save_only_latest, - args.save_every_weights, - args.cache_data_in_gpu, - args.overtraining_detector, - args.overtraining_threshold, - args.cleanup, - args.model_author, - args.vocoder, - args.checkpointing, - args.optimizer, - args.energy_use, - args.use_custom_reference, - args.reference_path, - args.multiscale_mel_loss, - args.use_cosine_annealing_lr, - args.architecture -) - -disc_version = version if vocoder not in ["RefineGAN", "BigVGAN"] else "v3" - -is_half = main_config.is_half -if main_config.brain: is_half = True - -if architecture == "SVC": - disc_version = version if vocoder != "Default" else "v0" - pitch_guidance = True - energy_use = False - -weights_path = main_configs["weights_path"] -logs_path = main_configs["logs_path"] -custom_save_checkpoint_path = None - -if not os.path.exists(model_name): experiment_dir = os.path.join(logs_path, model_name) -else: - experiment_dir = model_name - model_name = os.path.basename(model_name) - custom_save_checkpoint_path = weights_path - -checkpoint_path = experiment_dir if custom_save_checkpoint_path is None else custom_save_checkpoint_path - -training_file_path = os.path.join(experiment_dir, "training_data.json") -config_save_path = os.path.join(experiment_dir, "config.json") -filelist_path = os.path.join(experiment_dir, "filelist.txt") -eval_dir = os.path.join(experiment_dir, "eval") -spec_dirs = None - -d_lr_coeff = 1.0 -g_lr_coeff = 1.0 -d_step_per_g_step = 1 - -save_the_pid = True -cache_spectrogram = True -use_clip_grad_value = False - -torch.backends.cudnn.deterministic = args.deterministic if not main_config.device.startswith(("ocl", "privateuseone")) and not main_config.is_zluda else False -torch.backends.cudnn.benchmark = args.benchmark if not main_config.device.startswith(("ocl", "privateuseone")) and not main_config.is_zluda else False - -torch.backends.cuda.matmul.allow_tf32 = main_config.tf32 if not main_config.device.startswith(("ocl", "privateuseone")) and not main_config.is_zluda else False -torch.backends.cudnn.allow_tf32 = main_config.tf32 if not main_config.device.startswith(("ocl", "privateuseone")) and not main_config.is_zluda else False - -lowest_value = {"step": 0, "value": float("inf"), "epoch": 0} -global_step, last_loss_gen_all, overtrain_save_epoch = 0, 0, 0 -loss_gen_history, smoothed_loss_gen_history, loss_disc_history, smoothed_loss_disc_history = [], [], [], [] - -avg_losses = { - "grad_d_50": deque(maxlen=50), - "grad_g_50": deque(maxlen=50), - "disc_loss_50": deque(maxlen=50), - "adv_loss_50": deque(maxlen=50), - "fm_loss_50": deque(maxlen=50), - "kl_loss_50": deque(maxlen=50), - "mel_loss_50": deque(maxlen=50), - "gen_loss_50": deque(maxlen=50) -} - -with open(config_save_path, "r", encoding="utf-8") as f: - config = json.load(f) - -config = HParams(**config) -config.data.training_files = filelist_path - -def main(): - global training_file_path, last_loss_gen_all, smoothed_loss_gen_history, loss_gen_history, loss_disc_history, smoothed_loss_disc_history, overtrain_save_epoch, model_author, vocoder, checkpointing, gpus, energy_use - - log_data = { - translations["modelname"]: model_name, - translations["save_every_epoch"]: save_every_epoch, - translations["total_e"]: total_epoch, - translations["dorg"].format(pretrainG=pretrainG, pretrainD=pretrainD): "", - translations["training_version"]: version, - "Gpu": gpus, - translations["batch_size"]: batch_size, - translations["training_f0"]: pitch_guidance, - translations["save_only_latest"]: save_only_latest, - translations["save_every_weights"]: save_every_weights, - translations["cache_in_gpu"]: cache_data_in_gpu, - translations["overtraining_detector"]: overtraining_detector, - translations["threshold"]: overtraining_threshold, - translations["cleanup_training"]: cleanup, - translations["memory_efficient_training"]: checkpointing, - translations["optimizer"]: optimizer_choice, - translations["train&energy"]: energy_use, - translations["multiscale_mel_loss"]: multiscale_mel_loss, - translations["model_author"].format(model_author=model_author): "", - translations["vocoder"]: vocoder, - translations["cosine_annealing_lr"]: use_cosine_annealing_lr, - translations["architecture"]: architecture - } - - for key, value in log_data.items(): - logger.debug(f"{key}: {value}" if value != "" else f"{key} {value}") - - try: - os.environ["MASTER_ADDR"] = "localhost" - os.environ["MASTER_PORT"] = str(randint(20000, 55555)) - - wavs = glob.glob(os.path.join(os.path.join(experiment_dir, "sliced_audios"), "*.wav")) - if wavs: - _, sr = load_wav_to_torch(wavs[0]) - if sr != config.data.sample_rate: - logger.warning(translations["training_sr"].format(sr_1=config.data.sample_rate, sr_2=sr)) - sys.exit(1) - else: - logger.warning(translations["not_found_dataset"]) - sys.exit(1) - - if gpus == "-": - device, gpus = torch.device("cpu"), [0] - n_gpus = 1 - logger.warning(translations["not_gpu"]) - elif main_config.device.startswith("cuda"): - device, gpus = torch.device("cuda"), [int(item) for item in gpus.split("-")] - n_gpus = len(gpus) - elif main_config.device.startswith("xpu"): - device, gpus = torch.device("xpu"), [int(item) for item in gpus.split("-")] - n_gpus = len(gpus) - elif main_config.device.startswith("ocl"): - device, gpus = torch.device("ocl"), [int(item) for item in gpus.split("-")] - n_gpus = len(gpus) - elif main_config.device.startswith("privateuseone"): - device, gpus = torch.device("privateuseone"), [int(item) for item in gpus.split("-")] - n_gpus = len(gpus) - elif main_config.device.startswith("mps"): - device, gpus = torch.device("mps"), [0] - n_gpus = 1 - else: - device, gpus = torch.device("cpu"), [0] - n_gpus = 1 - logger.warning(translations["not_gpu"]) - - logger.info( - translations["use_precision"].format( - fp=("BF16" if main_config.brain else "FP16") if is_half else "FP32" - ) - ) - - def start(): - children = [] - pid_data = {"process_pids": []} - - if save_the_pid: - with open(config_save_path, "r", encoding="utf-8") as f: - try: - pid_data.update(json.load(f)) - except json.JSONDecodeError: - pass - - for rank, device_id in enumerate(gpus): - subproc = mp.Process( - target=run, - args=( - rank, - n_gpus, - pretrainG, - pretrainD, - pitch_guidance, - total_epoch, - save_every_weights, - config, - device, - device_id, - model_author, - vocoder, - checkpointing, - energy_use - ) - ) - - children.append(subproc) - subproc.start() - pid_data["process_pids"].append(subproc.pid) - - if save_the_pid: - with open(config_save_path, "w", encoding="utf-8") as f: - json.dump(pid_data, f, indent=4) - - for i in range(n_gpus): - children[i].join() - - def load_from_json(file_path): - if os.path.exists(file_path): - with open(file_path, "r") as f: - data = json.load(f) - - return ( - data.get("loss_disc_history", []), - data.get("smoothed_loss_disc_history", []), - data.get("loss_gen_history", []), - data.get("smoothed_loss_gen_history", []) - ) - - return [], [], [], [] - - def continue_overtrain_detector(training_file_path): - if overtraining_detector and os.path.exists(training_file_path): - ( - loss_disc_history, - smoothed_loss_disc_history, - loss_gen_history, - smoothed_loss_gen_history - )= load_from_json(training_file_path) - - if cleanup: - for root, dirs, files in os.walk(experiment_dir, topdown=False): - for name in files: - file_path = os.path.join(root, name) - file_name, file_extension = os.path.splitext(name) - - if ( - file_extension == ".0" or - (file_name.startswith(("D_", "G_")) and file_extension == ".pth") or - (file_name.startswith(("added", "trained")) and file_extension == ".index") - ): - os.remove(file_path) - - for name in dirs: - if name == "eval": - folder_path = os.path.join(root, name) - - for item in os.listdir(folder_path): - item_path = os.path.join(folder_path, item) - if os.path.isfile(item_path): os.remove(item_path) - - os.rmdir(folder_path) - - continue_overtrain_detector(training_file_path) - start() - except Exception as e: - logger.error(f"{translations['training_error']} {e}") - import traceback - logger.debug(traceback.format_exc()) - -class EpochRecorder: - def __init__(self): - self.last_time = ttime() - - def record(self): - now_time = ttime() - elapsed_time = now_time - self.last_time - self.last_time = now_time - - return translations["time_or_speed_training"].format( - current_time=datetime.datetime.now().strftime("%H:%M:%S"), - elapsed_time_str=str(datetime.timedelta(seconds=int(round(elapsed_time, 1)))) - ) - -def run( - rank, - n_gpus, - pretrainG, - pretrainD, - pitch_guidance, - custom_total_epoch, - custom_save_every_weights, - config, - device, - device_id, - model_author, - vocoder, - checkpointing, - energy_use -): - global global_step, smoothed_value_gen, smoothed_value_disc, optimizer_choice - - smoothed_value_gen, smoothed_value_disc = 0, 0 - - dist.init_process_group( - backend="gloo" if sys.platform == "win32" or device.type not in ["cuda", "xpu"] else ("xccl" if device.type == "xpu" else "nccl"), - init_method="env://", - world_size=n_gpus if device.type in ["cuda", "xpu"] else 1, - rank=rank if device.type in ["cuda", "xpu"] else 0 - ) - - torch.manual_seed(config.train.seed) - if device.type == "cuda": torch.cuda.manual_seed(config.train.seed) - elif device.type == "xpu": torch.xpu.manual_seed(config.train.seed) - elif device.type == "ocl": opencl.pytorch_ocl.manual_seed_all(config.train.seed) - - if device.type == "cuda": torch.cuda.set_device(device_id) - elif device.type == "xpu": torch.xpu.set_device(device_id) - - writer_eval = SummaryWriter( - log_dir=eval_dir - ) if rank == 0 else None - - from main.inference.training.data_utils import ( - DistributedBucketSampler, - TextAudioCollate, - TextAudioLoader - ) - - train_dataset = TextAudioLoader( - config.data, - spec_dirs=spec_dirs, - cache_spectrogram=cache_spectrogram, - pitch_guidance=pitch_guidance, - energy=energy_use - ) - - train_loader = DataLoader( - train_dataset, - num_workers=4, - shuffle=False, - pin_memory=True, - batch_size=1 if architecture != "SVC" else batch_size, - collate_fn=TextAudioCollate( - pitch_guidance=pitch_guidance, - energy=energy_use - ), - batch_sampler=DistributedBucketSampler( - train_dataset, - batch_size, - [50, 100, 200, 300, 400, 500, 600, 700, 800, 900], - num_replicas=n_gpus, - rank=rank, - shuffle=True - ) if architecture != "SVC" else None, - persistent_workers=True, - prefetch_factor=8 - ) - - if len(train_loader) < 3: - logger.warning(translations["not_enough_data"]) - sys.exit(1) - - spk_dim = config.model.spk_embed_dim - - try: - spk_dim = config.sid - except Exception as e: - logger.debug(e) - - try: - g_path = os.path.join(checkpoint_path, "G_latest.pth") - last_g = g_path if save_only_latest and os.path.exists(g_path) else latest_checkpoint_path(checkpoint_path, "G_*.pth") - - chk_path = (last_g if last_g else (pretrainG if pretrainG not in ["", "None"] else None)) - - if chk_path: - ckpt = torch.load(chk_path, map_location="cpu", weights_only=True) - spk_dim = ckpt["model"]["emb_g.weight"].shape[0] - del ckpt - except Exception as e: - logger.debug(e) - - config.model.spk_embed_dim = spk_dim - - from main.library.algorithm.discriminators import MultiPeriodDiscriminator - from main.library.algorithm.synthesizers import Synthesizer, SynthesizerSVC - - net_g, net_d = ( - ( - Synthesizer( - config.data.filter_length // 2 + 1, - config.train.segment_size // config.data.hop_length, - **config.model, - use_f0=pitch_guidance, - sr=config.data.sample_rate, - vocoder=vocoder, - checkpointing=checkpointing, - energy=energy_use - ) - ) if architecture == "RVC" else ( - SynthesizerSVC( - config.data.filter_length // 2 + 1, - config.train.segment_size // config.data.hop_length, - **config.model, - sr=config.data.sample_rate, - vocoder=vocoder, - checkpointing=checkpointing, - ) - ), - MultiPeriodDiscriminator( - version=disc_version, - use_spectral_norm=config.model.use_spectral_norm, - checkpointing=checkpointing - ) - ) - - net_g, net_d = ( - net_g.cuda(device_id), - net_d.cuda(device_id) - ) if device.type == "cuda" else ( - net_g.xpu(device_id), - net_d.xpu(device_id) - ) if device.type == "xpu" else ( - net_g.to(device), - net_d.to(device) - ) - - if optimizer_choice == "AnyPrecisionAdamW" and main_config.brain: - from main.library.optimizers.anyprecision_optimizer import AnyPrecisionAdamW - - optimizer_optim = AnyPrecisionAdamW - elif optimizer_choice == "RAdam": - from torch.optim import RAdam - - optimizer_optim = RAdam - elif optimizer_choice == "AdaBelief": - from main.library.optimizers.adabelief import AdaBelief - - optimizer_optim = AdaBelief - elif optimizer_choice == "AdaBeliefV2": - from main.library.optimizers.adabeliefv2 import AdaBeliefV2, get_inverse_sqrt_scheduler - - optimizer_optim = AdaBeliefV2 - else: - from torch.optim import AdamW - - optimizer_optim = AdamW - - optim_g, optim_d = ( - optimizer_optim( - net_g.parameters(), - config.train.learning_rate * g_lr_coeff, - betas=config.train.betas if not optimizer_choice.startswith("AdaBelief") else 1e-8, - eps=config.train.eps - ), - optimizer_optim( - net_d.parameters(), - config.train.learning_rate * d_lr_coeff, - betas=config.train.betas if not optimizer_choice.startswith("AdaBelief") else 1e-8, - eps=config.train.eps - ) - ) - - fn_mel_loss = MultiScaleMelSpectrogramLoss(sample_rate=config.data.sample_rate) if multiscale_mel_loss else torch.nn.L1Loss() - - if not device.type.startswith(("privateuseone", "ocl", "mps", "xpu")): - net_g, net_d = ( - DDP(net_g, device_ids=[device_id]), - DDP(net_d, device_ids=[device_id]) - ) if device.type.startswith("cuda") else ( - DDP(net_g), - DDP(net_d) - ) - - scaler_dict = {} - try: - if rank == 0: logger.info(translations["start_training"]) - - d_path = os.path.join(checkpoint_path, "D_latest.pth") if save_only_latest else latest_checkpoint_path(checkpoint_path, "D_*.pth") - g_path = os.path.join(checkpoint_path, "G_latest.pth") if save_only_latest else latest_checkpoint_path(checkpoint_path, "G_*.pth") - - _, _, _, epoch_str, scaler_dict = load_checkpoint( - logger, - d_path, - net_d, - optim_d - ) - - _, _, _, epoch_str, _ = load_checkpoint( - logger, - g_path, - net_g, - optim_g - ) - - if rank == 0: logger.info(translations["load_checkpoint"].format(d_path=d_path, g_path=g_path)) - - epoch_str += 1 - global_step = (epoch_str - 1) * len(train_loader) - except: - check = ["", "None"] - epoch_str, global_step = 1, 0 - strict = main_configs.get("pretrain_strict", True) - try: - if pretrainG not in check: - if rank == 0: logger.info(translations["import_pretrain"].format(dg="G", pretrain=pretrainG)) - - ckptG = torch.load(pretrainG, map_location="cpu", weights_only=True)["model"] - if architecture == "SVC" and "emb_g.weight" not in ckptG: ckptG["emb_g.weight"] = net_g.module.emb_g.weight if hasattr(net_g, "module") else net_g.emb_g.weight - net_g.module.load_state_dict(ckptG, strict=strict) if hasattr(net_g, "module") else net_g.load_state_dict(ckptG, strict=strict) - del ckptG - - if pretrainD not in check: - if rank == 0: logger.info(translations["import_pretrain"].format(dg="D", pretrain=pretrainD)) - - ckptD = torch.load(pretrainD, map_location="cpu", weights_only=True)["model"] - net_d.module.load_state_dict(ckptD, strict=strict) if hasattr(net_d, "module") else net_d.load_state_dict(ckptD, strict=strict) - del ckptD - except Exception as e: - logger.error(translations["checkpointing_err"]) - logger.debug(e) - sys.exit(1) - - if optimizer_choice == "AdaBelief" or use_cosine_annealing_lr: - scheduler_g, scheduler_d = ( - torch.optim.lr_scheduler.CosineAnnealingLR( - optim_g, - T_max=total_epoch, - eta_min=1e-6, - last_epoch=epoch_str - 2 - ), - torch.optim.lr_scheduler.CosineAnnealingLR( - optim_d, - T_max=total_epoch, - eta_min=1e-6, - last_epoch=epoch_str - 2 - ) - ) - elif optimizer_choice == "AdaBeliefV2": - scheduler_g, scheduler_d = ( - get_inverse_sqrt_scheduler( - optim_g, - warmup_epochs=10, - last_epoch=epoch_str - 2 - ), - get_inverse_sqrt_scheduler( - optim_d, - warmup_epochs=10, - last_epoch=epoch_str - 2 - ) - ) - else: - scheduler_g, scheduler_d = ( - torch.optim.lr_scheduler.ExponentialLR( - optim_g, - gamma=config.train.lr_decay, - last_epoch=epoch_str - 2 - ), - torch.optim.lr_scheduler.ExponentialLR( - optim_d, - gamma=config.train.lr_decay, - last_epoch=epoch_str - 2 - ) - ) - - if device.type == "xpu" and is_half: xpu.setup_gradscaler() - scaler = GradScaler(device=device, enabled=is_half and device.type in ["cuda", "xpu"]) - cache = [] - - if len(scaler_dict) > 0: scaler.load_state_dict(scaler_dict) - - if use_custom_reference and os.path.isfile(os.path.join(reference_path, "feats.npy")): - import numpy as np - - if rank == 0: logger.info(translations["using_reference"].format(reference_name=re.sub(r'_v\d+_(?:[A-Za-z0-9_]+?)_(True|False)_(True|False)$', '', os.path.basename(reference_path)))) - phone = np.repeat(np.load(os.path.join(reference_path, "feats.npy")), 2, axis=0) - - reference = ( - torch.FloatTensor(phone).unsqueeze(0).to(device), - torch.LongTensor([phone.shape[0]]).to(device), - torch.LongTensor(np.load(os.path.join(reference_path, "pitch_coarse.npy"))[:-1]).unsqueeze(0).to(device) if pitch_guidance else None, - torch.FloatTensor(np.load(os.path.join(reference_path, "pitch_fine.npy"))[:-1]).unsqueeze(0).to(device) if pitch_guidance else None, - torch.LongTensor([0]).to(device), - torch.FloatTensor(np.load(os.path.join(reference_path, "energy.npy"))[:-1]).unsqueeze(0).to(device) if energy_use else None, - ) - else: - info = next(iter(train_loader)) - reference = (info[0].to(device), info[1].to(device)) - - if pitch_guidance: reference += (info[2].to(device), info[3].to(device), info[8].to(device), info[9].to(device) if energy_use else None) - else: reference += (None, None, info[6].to(device), info[7].to(device) if energy_use else None) - - for epoch in range(epoch_str, total_epoch + 1): - train_and_evaluate( - rank, - epoch, - config, - [net_g, net_d], - [optim_g, optim_d], - scaler, - train_loader, - writer_eval, - cache, - custom_save_every_weights, - custom_total_epoch, - device, - device_id, - reference, - model_author, - vocoder, - energy_use, - fn_mel_loss - ) - - scheduler_g.step(); scheduler_d.step() - -def train_and_evaluate( - rank, - epoch, - hps, - nets, - optims, - scaler, - train_loader, - writer, - cache, - custom_save_every_weights, - custom_total_epoch, - device, - device_id, - reference, - model_author, - vocoder, - energy_use, - fn_mel_loss -): - global global_step, lowest_value, loss_disc, consecutive_increases_gen, consecutive_increases_disc, smoothed_value_gen, smoothed_value_disc - - if epoch == 1: - lowest_value = {"step": 0, "value": float("inf"), "epoch": 0} - consecutive_increases_gen, consecutive_increases_disc = 0, 0 - - net_g, net_d = nets - optim_g, optim_d = optims - - if architecture != "SVC": train_loader.batch_sampler.set_epoch(epoch) - net_g.train(); net_d.train() - - if device.type == "cuda" and cache_data_in_gpu: - data_iterator = cache - - if cache == []: - for batch_idx, info in enumerate(train_loader): - cache.append( - (batch_idx, [ - tensor.cuda(device_id, non_blocking=True) - for tensor in info - ]) - ) - else: - shuffle(cache) - elif device.type == "xpu" and cache_data_in_gpu: - data_iterator = cache - - if cache == []: - for batch_idx, info in enumerate(train_loader): - cache.append( - (batch_idx, [ - tensor.xpu(device_id, non_blocking=True) - for tensor in info - ]) - ) - else: - shuffle(cache) - elif device.type in ["privateuseone", "ocl"] and cache_data_in_gpu: - data_iterator = cache - - if cache == []: - for batch_idx, info in enumerate(train_loader): - cache.append( - (batch_idx, [ - tensor.to(device_id if device.type == "ocl" else device, non_blocking=True) - for tensor in info - ]) - ) - else: - shuffle(cache) - else: - data_iterator = enumerate(train_loader) - - epoch_recorder = EpochRecorder() - - autocast_enabled = is_half and device.type in ["cuda", "xpu"] - autocast_dtype = ( - torch.float32 - if not autocast_enabled else - (torch.bfloat16 if main_config.brain else torch.float16) - ) - - autocasts = autocast( - device.type, - enabled=autocast_enabled, - dtype=autocast_dtype - ) if not device.type.startswith(("ocl", "privateuseone")) else nullcontext() - - with tqdm(total=len(train_loader), leave=False) as pbar: - for batch_idx, info in data_iterator: - if device.type == "cuda" and not cache_data_in_gpu: - info = [ - tensor.cuda(device_id, non_blocking=True) - for tensor in info - ] - elif device.type == "xpu" and not cache_data_in_gpu: - info = [ - tensor.xpu(device_id, non_blocking=True) - for tensor in info - ] - elif device.type in ["privateuseone", "ocl"] and not cache_data_in_gpu: - info = [ - tensor.to(device_id if device.type == "ocl" else device, non_blocking=True) - for tensor in info - ] - else: - info = [ - tensor.to(device) - for tensor in info - ] - - phone, phone_lengths = info[0], info[1] - if pitch_guidance: - pitch, pitchf = info[2], info[3] - spec, spec_lengths, wave, sid = info[4], info[5], info[6], info[8] - energy = info[9] if energy_use else None - else: - pitch = pitchf = None - spec, spec_lengths, wave, sid = info[2], info[3], info[4], info[6] - energy = info[7] if energy_use else None - - with autocasts: - y_hat, ids_slice, _, z_mask, (_, z_p, m_p, logs_p, _, logs_q) = net_g( - phone, - phone_lengths, - pitch, - pitchf, - spec, - spec_lengths, - sid, - energy - ) - - wave = commons.slice_segments( - wave, - ids_slice * config.data.hop_length, - config.train.segment_size, - dim=3 - ) - - for _ in range(d_step_per_g_step): - with autocasts: - y_d_hat_r, y_d_hat_g, _, _ = net_d( - wave, - y_hat.detach() - ) - - loss_disc, losses_disc_r, losses_disc_g = losses.discriminator_loss( - y_d_hat_r, - y_d_hat_g - ) - - optim_d.zero_grad() - - if autocast_enabled: - scaler.scale(loss_disc).backward() - scaler.unscale_(optim_d) - grad_norm_d = commons.clip_grad_value(net_d.parameters(), None) if use_clip_grad_value else commons.grad_norm(net_d.parameters()) - scaler.step(optim_d) - else: - loss_disc.backward() - grad_norm_d = commons.clip_grad_value(net_d.parameters(), None) if use_clip_grad_value else commons.grad_norm(net_d.parameters()) - optim_d.step() - - with autocasts: - y_d_hat_r, y_d_hat_g, fmap_r, fmap_g = net_d( - wave, - y_hat - ) - - if multiscale_mel_loss: - loss_mel = fn_mel_loss( - wave, - y_hat - ) * config.train.c_mel / 3.0 - else: - y_hat_mel = mel_spectrogram_torch( - y_hat.float().squeeze(1), - config.data.filter_length, - config.data.n_mel_channels, - config.data.sample_rate, - config.data.hop_length, - config.data.win_length, - config.data.mel_fmin, - config.data.mel_fmax - ) - - loss_mel = fn_mel_loss( - mel_spectrogram_torch( - wave.float().squeeze(1), - config.data.filter_length, - config.data.n_mel_channels, - config.data.sample_rate, - config.data.hop_length, - config.data.win_length, - config.data.mel_fmin, - config.data.mel_fmax - ), - y_hat_mel - ) * config.train.c_mel - - if device.type == "privateuseone": - loss_kl = ( - losses.kl_loss( - z_p.detach().cpu(), - logs_q.detach().cpu(), - m_p.detach().cpu(), - logs_p.detach().cpu(), - z_mask.detach().cpu() - ) * config.train.c_kl - ).to(device) - else: - loss_kl = losses.kl_loss( - z_p, - logs_q, - m_p, - logs_p, - z_mask - ) * config.train.c_kl - - loss_fm = losses.feature_loss(fmap_r, fmap_g) - loss_gen, losses_gen = losses.generator_loss(y_d_hat_g) - loss_gen_all = loss_gen + loss_fm + loss_mel + loss_kl - - if loss_gen_all < lowest_value["value"]: - lowest_value = { - "step": global_step, - "value": loss_gen_all, - "epoch": epoch - } - - optim_g.zero_grad() - if autocast_enabled: - scaler.scale(loss_gen_all).backward() - scaler.unscale_(optim_g) - grad_norm_g = commons.clip_grad_value(net_g.parameters(), None) if use_clip_grad_value else commons.grad_norm(net_g.parameters()) - scaler.step(optim_g) - scaler.update() - else: - loss_gen_all.backward() - grad_norm_g = commons.clip_grad_value(net_g.parameters(), None) if use_clip_grad_value else commons.grad_norm(net_g.parameters()) - optim_g.step() - - global_step += 1 - - avg_losses["grad_d_50"].append(grad_norm_d) - avg_losses["grad_g_50"].append(grad_norm_g) - avg_losses["disc_loss_50"].append(loss_disc.detach()) - avg_losses["adv_loss_50"].append(loss_gen.detach()) - avg_losses["fm_loss_50"].append(loss_fm.detach()) - avg_losses["kl_loss_50"].append(loss_kl.detach()) - avg_losses["mel_loss_50"].append(loss_mel.detach()) - avg_losses["gen_loss_50"].append(loss_gen_all.detach()) - - if rank == 0 and global_step % 50 == 0: - scalar_dict = { - "grad_avg_50/norm_d": sum(avg_losses["grad_d_50"]) / len(avg_losses["grad_d_50"]), - "grad_avg_50/norm_g": sum(avg_losses["grad_g_50"]) / len(avg_losses["grad_g_50"]), - "loss_avg_50/d/adv": torch.stack(list(avg_losses["disc_loss_50"])).mean(), - "loss_avg_50/g/adv": torch.stack(list(avg_losses["adv_loss_50"])).mean(), - "loss_avg_50/g/fm": torch.stack(list(avg_losses["fm_loss_50"])).mean(), - "loss_avg_50/g/kl": torch.stack(list(avg_losses["kl_loss_50"])).mean(), - "loss_avg_50/g/mel": torch.stack(list(avg_losses["mel_loss_50"])).mean(), - "loss_avg_50/g/total": torch.stack(list(avg_losses["gen_loss_50"])).mean() - } - - summarize( - writer=writer, - global_step=global_step, - scalars=scalar_dict - ) - - pbar.update(1) - - with torch.no_grad(): - clear_gpu_cache() - - if rank == 0: - mel = spec_to_mel_torch( - spec, - config.data.filter_length, - config.data.n_mel_channels, - config.data.sample_rate, - config.data.mel_fmin, - config.data.mel_fmax - ) - - y_mel = commons.slice_segments( - mel, - ids_slice, - config.train.segment_size // config.data.hop_length, - dim=3 - ) - - y_hat_mel = mel_spectrogram_torch( - y_hat.float().squeeze(1), - config.data.filter_length, - config.data.n_mel_channels, - config.data.sample_rate, - config.data.hop_length, - config.data.win_length, - config.data.mel_fmin, - config.data.mel_fmax - ) - - scalar_dict = { - "loss/g/total": loss_gen_all, - "loss/d/adv": loss_disc, - "learning_rate": optim_g.param_groups[0]["lr"], - "grad/norm_d": grad_norm_d, - "grad/norm_g": grad_norm_g, - "loss/g/adv": loss_gen, - "loss/g/fm": loss_fm, - "loss/g/mel": loss_mel, - "loss/g/kl": loss_kl - } - - scalar_dict.update({f"loss/g/{i}": v for i, v in enumerate(losses_gen)}) - scalar_dict.update({f"loss/d_r/{i}": v for i, v in enumerate(losses_disc_r)}) - scalar_dict.update({f"loss/d_g/{i}": v for i, v in enumerate(losses_disc_g)}) - - image_dict = { - "slice/mel_org": plot_spectrogram_to_numpy(y_mel[0].data.cpu().numpy()), - "slice/mel_gen": plot_spectrogram_to_numpy(y_hat_mel[0].data.cpu().numpy()), - "all/mel": plot_spectrogram_to_numpy(mel[0].data.cpu().numpy()), - } - - if epoch % save_every_epoch == 0: - with autocasts: - with torch.no_grad(): - o, *_ = net_g.module.infer(*reference) if hasattr(net_g, "module") else net_g.infer(*reference) - - summarize( - writer=writer, - global_step=global_step, - images=image_dict, - scalars=scalar_dict, - audios={ - f"gen/audio_{global_step:07d}": o[0, :, :] - }, - audio_sample_rate=config.data.sample_rate - ) - else: - summarize( - writer=writer, - global_step=global_step, - images=image_dict, - scalars=scalar_dict - ) - - def check_overtraining( - smoothed_loss_history, - threshold, - epsilon=0.004 - ): - if len(smoothed_loss_history) < threshold + 1: return False - - for i in range(-threshold, -1): - if smoothed_loss_history[i + 1] > smoothed_loss_history[i]: - return True - - if abs(smoothed_loss_history[i + 1] - smoothed_loss_history[i]) >= epsilon: - return False - - return True - - def update_exponential_moving_average( - smoothed_loss_history, - new_value, - smoothing=0.987 - ): - smoothed_value = ( - new_value - if not smoothed_loss_history else - (smoothing * smoothed_loss_history[-1] + (1 - smoothing) * new_value) - ) - - smoothed_loss_history.append(smoothed_value) - return smoothed_value - - def save_to_json( - file_path, - loss_disc_history, - smoothed_loss_disc_history, - loss_gen_history, - smoothed_loss_gen_history - ): - with open(file_path, "w", encoding="utf-8") as f: - json.dump({ - "loss_disc_history": loss_disc_history, - "smoothed_loss_disc_history": smoothed_loss_disc_history, - "loss_gen_history": loss_gen_history, - "smoothed_loss_gen_history": smoothed_loss_gen_history - }, f) - - model_add, model_del = [], [] - done = False - - if rank == 0: - if epoch % save_every_epoch == False: - checkpoint_suffix = f"{'latest' if save_only_latest else global_step}.pth" - - save_checkpoint( - logger, - net_g, - optim_g, - config.train.learning_rate, - epoch, - os.path.join(checkpoint_path, "G_" + checkpoint_suffix), - scaler - ) - save_checkpoint( - logger, - net_d, - optim_d, - config.train.learning_rate, - epoch, - os.path.join(checkpoint_path, "D_" + checkpoint_suffix), - scaler - ) - - if custom_save_every_weights: - model_add.append( - os.path.join(weights_path, f"{model_name}_{epoch}e_{global_step}s.pth") - ) - - if overtraining_detector and epoch > 1: - current_loss_disc, current_loss_gen = float(loss_disc), float(lowest_value["value"]) - - loss_disc_history.append(current_loss_disc) - loss_gen_history.append(current_loss_gen) - - smoothed_value_disc = update_exponential_moving_average( - smoothed_loss_disc_history, - current_loss_disc - ) - - smoothed_value_gen = update_exponential_moving_average( - smoothed_loss_gen_history, - current_loss_gen - ) - - is_overtraining_disc = check_overtraining( - smoothed_loss_disc_history, - overtraining_threshold * 2 - ) - - is_overtraining_gen = check_overtraining( - smoothed_loss_gen_history, - overtraining_threshold, - 0.01 - ) - - consecutive_increases_disc = (consecutive_increases_disc + 1) if is_overtraining_disc else 0 - consecutive_increases_gen = (consecutive_increases_gen + 1) if is_overtraining_gen else 0 - - if epoch % save_every_epoch == 0: - save_to_json( - training_file_path, - loss_disc_history, - smoothed_loss_disc_history, - loss_gen_history, - smoothed_loss_gen_history - ) - - if ( - is_overtraining_gen and - consecutive_increases_gen == overtraining_threshold or - is_overtraining_disc and - consecutive_increases_disc == (overtraining_threshold * 2) - ): - logger.info( - translations["overtraining_find"].format( - epoch=epoch, - smoothed_value_gen=f"{smoothed_value_gen:.3f}", - smoothed_value_disc=f"{smoothed_value_disc:.3f}" - ) - ) - - done = True - else: - logger.info( - translations["best_epoch"].format( - epoch=epoch, - smoothed_value_gen=f"{smoothed_value_gen:.3f}", - smoothed_value_disc=f"{smoothed_value_disc:.3f}" - ) - ) - - for file in glob.glob(os.path.join(weights_path, f"{model_name}_*e_*s_best_epoch.pth")): - model_del.append(file) - - model_add.append( - os.path.join(weights_path, f"{model_name}_{epoch}e_{global_step}s_best_epoch.pth") - ) - - if epoch >= custom_total_epoch: - logger.info( - translations["success_training"].format( - epoch=epoch, - global_step=global_step, - loss_gen_all=round(loss_gen_all.item(), 3) - ) - ) - - logger.info( - translations["training_info"].format( - lowest_value_rounded=round(float(lowest_value["value"]), 3), - lowest_value_epoch=lowest_value['epoch'], - lowest_value_step=lowest_value['step'] - ) - ) - - model_add.append( - os.path.join(weights_path, f"{model_name}_{epoch}e_{global_step}s.pth") - ) - - done = True - - for m in model_del: - os.remove(m) - - if model_add: - ckpt = (net_g.module.state_dict() if hasattr(net_g, "module") else net_g.state_dict()) - for m in model_add: - extract_model( - ckpt=ckpt, - sr=config.data.sample_rate, - pitch_guidance=pitch_guidance == True, - name=model_name, - model_path=m, - epoch=epoch, - step=global_step, - version=version, - hps=hps, - model_author=model_author, - vocoder=vocoder, - energy_use=energy_use, - speakers_id=config.sid, - architecture=architecture - ) - - lowest_value_rounded = round(float(lowest_value["value"]), 3) - - if epoch > 1 and overtraining_detector: - logger.info( - translations["model_training_info"].format( - model_name=model_name, - epoch=epoch, - global_step=global_step, - epoch_recorder=epoch_recorder.record(), - lowest_value_rounded=lowest_value_rounded, - lowest_value_epoch=lowest_value['epoch'], - lowest_value_step=lowest_value['step'], - remaining_epochs_gen=(overtraining_threshold - consecutive_increases_gen), - remaining_epochs_disc=((overtraining_threshold * 2) - consecutive_increases_disc), - smoothed_value_gen=f"{smoothed_value_gen:.3f}", - smoothed_value_disc=f"{smoothed_value_disc:.3f}" - ) - ) - elif epoch > 1 and overtraining_detector == False: - logger.info( - translations["model_training_info_2"].format( - model_name=model_name, - epoch=epoch, - global_step=global_step, - epoch_recorder=epoch_recorder.record(), - lowest_value_rounded=lowest_value_rounded, - lowest_value_epoch=lowest_value['epoch'], - lowest_value_step=lowest_value['step'] - ) - ) - else: - logger.info( - translations["model_training_info_3"].format( - model_name=model_name, - epoch=epoch, - global_step=global_step, - epoch_recorder=epoch_recorder.record() - ) - ) - - logger.debug( - f"loss_gen_all: {loss_gen_all} loss_gen: {loss_gen} loss_fm: {loss_fm} loss_mel: {loss_mel} loss_kl: {loss_kl}" - ) - - last_loss_gen_all = loss_gen_all - - if done: - if save_the_pid: - pid_file_path = os.path.join(experiment_dir, "config.json") - - with open(pid_file_path, "r", encoding="utf-8") as pid_file: - pid_data = json.load(pid_file) - - with open(pid_file_path, "w", encoding="utf-8") as pid_file: - pid_data.pop("process_pids", None) - json.dump(pid_data, pid_file, indent=4) - - if os.path.exists(os.path.join(experiment_dir, "train_pid.txt")): - os.remove(os.path.join(experiment_dir, "train_pid.txt")) - - sys.exit(0) - - with torch.no_grad(): - clear_gpu_cache() - -if __name__ == "__main__": - mp.set_start_method("spawn") - main() \ No newline at end of file diff --git a/main/inference/training/utils.py b/main/inference/training/utils.py deleted file mode 100644 index e8f2a6f72e965db82d1ff6a57dc0a62d3ad07c09..0000000000000000000000000000000000000000 --- a/main/inference/training/utils.py +++ /dev/null @@ -1,196 +0,0 @@ -import os -import sys -import glob -import torch - -import numpy as np -import soundfile as sf -import matplotlib.pyplot as plt - -from collections import OrderedDict - -sys.path.append(os.getcwd()) - -from main.app.variables import config, translations - -MATPLOTLIB_FLAG = False - -def optimizer_state_dict_cpu(optimizer): - import copy - - opt_state = copy.deepcopy(optimizer.state_dict()) - - for state in opt_state["state"].values(): - for k, v in state.items(): - if torch.is_tensor(v): - state[k] = v.detach().cpu() - - return opt_state - -def replace_keys_in_dict(d, old_key_part, new_key_part): - updated_dict = OrderedDict() if isinstance(d, OrderedDict) else {} - - for key, value in d.items(): - updated_dict[( - key.replace(old_key_part, new_key_part) if isinstance(key, str) else key - )] = ( - replace_keys_in_dict(value, old_key_part, new_key_part) if isinstance(value, dict) else value - ) - - return updated_dict - -def load_checkpoint(logger, checkpoint_path, model, optimizer=None, load_opt=1): - assert os.path.isfile(checkpoint_path), translations["not_found_checkpoint"].format(checkpoint_path=checkpoint_path) - - checkpoint_dict = replace_keys_in_dict( - replace_keys_in_dict( - torch.load(checkpoint_path, map_location="cpu", weights_only=True), - ".weight_v", - ".parametrizations.weight.original1" - ), - ".weight_g", - ".parametrizations.weight.original0" - ) - - new_state_dict = { - k: checkpoint_dict["model"].get(k, v) - for k, v in (model.module.state_dict() if hasattr(model, "module") else model.state_dict()).items() - } - - model.module.load_state_dict(new_state_dict, strict=False) if hasattr(model, "module") else model.load_state_dict(new_state_dict, strict=False) - if optimizer and load_opt == 1: optimizer.load_state_dict(checkpoint_dict.get("optimizer", {})) - logger.debug(translations["save_checkpoint"].format(checkpoint_path=checkpoint_path, checkpoint_dict=checkpoint_dict['iteration'])) - - return ( - model, - optimizer, - checkpoint_dict.get("learning_rate", 0), - checkpoint_dict["iteration"], - checkpoint_dict.get("scaler", {}) - ) - -def save_checkpoint(logger, model, optimizer, learning_rate, iteration, checkpoint_path, scaler): - state_dict = (model.module.state_dict() if hasattr(model, "module") else model.state_dict()) - - if config.device.startswith("privateuseone"): - model_state = {k: v.detach().cpu() for k, v in state_dict.items()} - model_optimizer = optimizer_state_dict_cpu(optimizer) - else: - model_state = state_dict - model_optimizer = optimizer.state_dict() - - torch.save( - replace_keys_in_dict( - replace_keys_in_dict({ - "model": model_state, - "iteration": iteration, - "optimizer": model_optimizer, - "learning_rate": learning_rate, - "scaler": scaler.state_dict() - }, ".parametrizations.weight.original1", ".weight_v"), - ".parametrizations.weight.original0", ".weight_g" - ), - checkpoint_path - ) - - logger.info(translations["save_model"].format(checkpoint_path=checkpoint_path, iteration=iteration)) - -def summarize( - writer, - global_step, - scalars={}, - histograms={}, - images={}, - audios={}, - audio_sample_rate=22050 -): - for k, v in scalars.items(): - writer.add_scalar(k, v, global_step) - - for k, v in histograms.items(): - writer.add_histogram(k, v, global_step) - - for k, v in images.items(): - writer.add_image(k, v, global_step, dataformats="HWC") - - for k, v in audios.items(): - writer.add_audio(k, v, global_step, audio_sample_rate) - -def latest_checkpoint_path(dir_path, regex="G_*.pth"): - checkpoints = sorted( - glob.glob( - os.path.join(dir_path, regex) - ), - key=lambda f: int("".join(filter(str.isdigit, f))) - ) - return checkpoints[-1] if checkpoints else None - -def plot_spectrogram_to_numpy(spectrogram): - global MATPLOTLIB_FLAG - - if not MATPLOTLIB_FLAG: - plt.switch_backend("Agg") - MATPLOTLIB_FLAG = True - - fig, ax = plt.subplots(figsize=(10, 2)) - plt.colorbar(ax.imshow(spectrogram, aspect="auto", origin="lower", interpolation="none"), ax=ax) - plt.xlabel("Frames") - plt.ylabel("Channels") - plt.tight_layout() - fig.canvas.draw() - plt.close(fig) - - try: - data = np.array( - fig.canvas.renderer.buffer_rgba(), - dtype=np.uint8 - ).reshape( - fig.canvas.get_width_height()[::-1] + (4,) - )[:, :, :3] - except: - data = np.fromstring( - fig.canvas.tostring_rgb(), - dtype=np.uint8, - sep="" - ).reshape( - fig.canvas.get_width_height()[::-1] + (3,) - ) - - return data - -def load_wav_to_torch(full_path): - data, sample_rate = sf.read(full_path, dtype=np.float32) - return torch.FloatTensor(data.astype(np.float32)), sample_rate - -def load_filepaths_and_text(filename, split="|"): - with open(filename, encoding="utf-8") as f: - return [line.strip().split(split) for line in f] - -class HParams: - def __init__(self, **kwargs): - for k, v in kwargs.items(): - self[k] = HParams(**v) if isinstance(v, dict) else v - - def keys(self): - return self.__dict__.keys() - - def items(self): - return self.__dict__.items() - - def values(self): - return self.__dict__.values() - - def __len__(self): - return len(self.__dict__) - - def __getitem__(self, key): - return self.__dict__[key] - - def __setitem__(self, key, value): - self.__dict__[key] = value - - def __contains__(self, key): - return key in self.__dict__ - - def __repr__(self): - return repr(self.__dict__) \ No newline at end of file diff --git a/main/library/algorithm/attentions.py b/main/library/algorithm/attentions.py deleted file mode 100644 index 3513b03d477054bf954afad71e6ce9b0c50a40d9..0000000000000000000000000000000000000000 --- a/main/library/algorithm/attentions.py +++ /dev/null @@ -1,318 +0,0 @@ -import os -import sys -import math -import torch - -import torch.nn as nn -import torch.nn.functional as F - -sys.path.append(os.getcwd()) - -from main.library.algorithm.commons import convert_pad_shape - -class MultiHeadAttention(nn.Module): - def __init__( - self, - channels, - out_channels, - n_heads, - p_dropout=0.0, - window_size=None, - heads_share=True, - block_length=None, - proximal_bias=False, - proximal_init=False, - onnx=False - ): - super().__init__() - assert channels % n_heads == 0 - self.channels = channels - self.out_channels = out_channels - self.n_heads = n_heads - self.k_channels = channels // n_heads - self.window_size = window_size - self.block_length = block_length - self.proximal_bias = proximal_bias - self.conv_q = nn.Conv1d(channels, channels, 1) - self.conv_k = nn.Conv1d(channels, channels, 1) - self.conv_v = nn.Conv1d(channels, channels, 1) - self.conv_o = nn.Conv1d(channels, out_channels, 1) - self.drop = nn.Dropout(p_dropout) - - self.pad_fn = self.padding if onnx else F.pad - self._get_relative_embeddings = self._get_relative_embeddings_onnx if onnx else self._get_relative_embeddings_torch - self._relative_position_to_absolute_position = self._relative_position_to_absolute_position_onnx if onnx else self._relative_position_to_absolute_position_torch - self._absolute_position_to_relative_position = self._absolute_position_to_relative_position_onnx if onnx else self._absolute_position_to_relative_position_torch - - if window_size is not None: - n_heads_rel = 1 if heads_share else n_heads - rel_stddev = self.k_channels**-0.5 - - self.emb_rel_k = nn.Parameter(torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels) * rel_stddev) - self.emb_rel_v = nn.Parameter(torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels) * rel_stddev) - - nn.init.xavier_uniform_(self.conv_q.weight) - nn.init.xavier_uniform_(self.conv_k.weight) - nn.init.xavier_uniform_(self.conv_v.weight) - nn.init.xavier_uniform_(self.conv_o.weight) - - if proximal_init: - with torch.no_grad(): - self.conv_k.weight.copy_(self.conv_q.weight) - self.conv_k.bias.copy_(self.conv_q.bias) - - def forward(self, x, c, attn_mask=None): - q, k, v = self.conv_q(x), self.conv_k(c), self.conv_v(c) - x, self.attn = self.attention(q, k, v, mask=attn_mask) - - return self.conv_o(x) - - def attention(self, query, key, value, mask=None): - b, d, t_s, t_t = (*key.size(), query.size(2)) - query = query.view(b, self.n_heads, self.k_channels, t_t).transpose(2, 3) - key = key.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3) - value = value.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3) - scores = (query / math.sqrt(self.k_channels)) @ key.transpose(-2, -1) - - if self.window_size: - assert (t_s == t_t) - scores += self._relative_position_to_absolute_position( - self._matmul_with_relative_keys( - query / math.sqrt(self.k_channels), - self._get_relative_embeddings( - self.emb_rel_k, - t_s - ) - ) - ) - - if self.proximal_bias: - assert t_s == t_t - scores += self._attention_bias_proximal(t_s).to(device=scores.device, dtype=scores.dtype) - - if mask is not None: - scores = scores.masked_fill(mask == 0, -1e4) - - if self.block_length: - assert (t_s == t_t) - block_mask = ( - torch.ones_like(scores) - .triu(-self.block_length) - .tril(self.block_length) - ) - scores = scores.masked_fill(block_mask == 0, -1e4) - - p_attn = self.drop(F.softmax(scores, dim=-1)) - output = p_attn @ value - - if self.window_size: - output += self._matmul_with_relative_values( - self._absolute_position_to_relative_position( - p_attn - ), - self._get_relative_embeddings( - self.emb_rel_v, - t_s - ) - ) - - return output.transpose(2, 3).contiguous().view(b, d, t_t), p_attn - - def _matmul_with_relative_values(self, x, y): - return x @ y.unsqueeze(0) - - def _matmul_with_relative_keys(self, x, y): - return x @ y.unsqueeze(0).transpose(-2, -1) - - def _get_relative_embeddings_onnx(self, relative_embeddings, length): - pad_length = (length - (self.window_size + 1)).clamp(min=0) - slice_start_position = ((self.window_size + 1) - length).clamp(min=0) - pad_shape = [0, 0, pad_length, pad_length, 0, 0] - - if pad_length > 0: - relative_embeddings = self.pad_fn( - relative_embeddings, - pad_shape - ) - - return relative_embeddings[:, slice_start_position:(slice_start_position + 2 * length - 1)] - - def _get_relative_embeddings_torch(self, relative_embeddings, length): - pad_length = max(length - (self.window_size + 1), 0) - slice_start_position = max((self.window_size + 1) - length, 0) - pad_shape = convert_pad_shape([[0, 0], [pad_length, pad_length], [0, 0]]) - - if pad_length > 0: - relative_embeddings = self.pad_fn( - relative_embeddings, - pad_shape - ) - - return relative_embeddings[:, slice_start_position:(slice_start_position + 2 * length - 1)] - - def _relative_position_to_absolute_position_onnx(self, x): - batch, heads, length, _ = x.size() - - pad = [0, 1] - pad_shape = [0, length - 1] - - pad = self.pad_fn(x, pad).view([batch, heads, length * 2 * length]) - return self.pad_fn(pad, pad_shape).view([batch, heads, length + 1, 2 * length - 1])[:, :, :length, length - 1 :] - - def _relative_position_to_absolute_position_torch(self, x): - batch, heads, length, _ = x.size() - - pad = convert_pad_shape([[0, 0], [0, 0], [0, 0], [0, 1]]) - pad_shape = convert_pad_shape([[0, 0], [0, 0], [0, length - 1]]) - - pad = self.pad_fn(x, pad).view([batch, heads, length * 2 * length]) - return self.pad_fn(pad, pad_shape).view([batch, heads, length + 1, 2 * length - 1])[:, :, :length, length - 1 :] - - def _absolute_position_to_relative_position_onnx(self, x): - batch, heads, length, _ = x.size() - - pad = [0, length - 1] - pad_shape = [length, 0] - - pad = self.pad_fn(x, pad).view([batch, heads, length**2 + length * (length - 1)]) - return self.pad_fn(pad, pad_shape).view([batch, heads, length, 2 * length])[:, :, :, 1:] - - def _absolute_position_to_relative_position_torch(self, x): - batch, heads, length, _ = x.size() - - pad = convert_pad_shape([[0, 0], [0, 0], [0, 0], [0, length - 1]]) - pad_shape = convert_pad_shape([[0, 0], [0, 0], [length, 0]]) - - pad = self.pad_fn(x, pad).view([batch, heads, length**2 + length * (length - 1)]) - return self.pad_fn(pad, pad_shape).view([batch, heads, length, 2 * length])[:, :, :, 1:] - - def _attention_bias_proximal(self, length): - r = torch.arange(length, dtype=torch.float32) - diff = r.unsqueeze(0) - r.unsqueeze(1) - return -diff.abs().log1p().unsqueeze(0).unsqueeze(0) - - def padding(self, x, pad): - if len(pad) == 6: - _, _, pad_left, pad_right, _, _ = pad - - if pad_left > 0 or pad_right > 0: - zeros = torch.zeros( - x.size(0), - pad_left, - x.size(2), - device=x.device, - dtype=x.dtype - ) - - x = torch.cat( - [zeros, x], - dim=1 - ) - - zeros = torch.zeros( - x.size(0), - pad_right, - x.size(2), - device=x.device, - dtype=x.dtype - ) - - x = torch.cat( - [x, zeros], - dim=1 - ) - else: - left, right = pad[-2], pad[-1] - - if left > 0: - zeros = torch.zeros( - *x.shape[:-1], - left, - device=x.device, - dtype=x.dtype - ) - - x = torch.cat( - [zeros, x], - dim=-1 - ) - - if right > 0: - zeros = torch.zeros( - *x.shape[:-1], - right, - device=x.device, - dtype=x.dtype - ) - - x = torch.cat( - [x, zeros], - dim=-1 - ) - - return x - -class FFN(nn.Module): - def __init__( - self, - in_channels, - out_channels, - filter_channels, - kernel_size, - p_dropout=0.0, - activation=None, - causal=False, - onnx=False - ): - super().__init__() - self.conv_1 = nn.Conv1d(in_channels, filter_channels, kernel_size) - self.conv_2 = nn.Conv1d(filter_channels, out_channels, kernel_size) - self.drop = nn.Dropout(p_dropout) - self.activation = activation - - self._causal_padding = self._causal_padding_onnx if onnx else self._causal_padding_torch - self._same_padding = self._same_padding_onnx if onnx else self._same_padding_torch - self.padding_fn = self._causal_padding if causal else self._same_padding - - def forward(self, x, x_mask): - x = self.conv_1(self.padding_fn(x * x_mask)) - x = self.drop(self._apply_activation(x)) - x = self.conv_2(self.padding_fn(x * x_mask)) - - return x * x_mask - - def _apply_activation(self, x): - if self.activation == "gelu": return x * (1.702 * x).sigmoid() - return x.relu() - - def _causal_padding_onnx(self, x): - pad_l, pad_r = self.conv_1.kernel_size[0] - 1, 0 - - return F.pad( - x, - [pad_l, pad_r, 0, 0, 0, 0] - ) - - def _causal_padding_torch(self, x): - pad_l, pad_r = self.conv_1.kernel_size[0] - 1, 0 - - return F.pad( - x, - convert_pad_shape([[0, 0], [0, 0], [pad_l, pad_r]]) - ) - - def _same_padding_onnx(self, x): - pad = (self.conv_1.kernel_size[0] - 1) // 2 - - return F.pad( - x, - [pad, pad, 0, 0, 0, 0] - ) - - def _same_padding_torch(self, x): - pad = (self.conv_1.kernel_size[0] - 1) // 2 - - return F.pad( - x, - convert_pad_shape([[0, 0], [0, 0], [pad, pad]]) - ) \ No newline at end of file diff --git a/main/library/algorithm/commons.py b/main/library/algorithm/commons.py deleted file mode 100644 index 464be92a880146208697d249e05699d66dcea171..0000000000000000000000000000000000000000 --- a/main/library/algorithm/commons.py +++ /dev/null @@ -1,72 +0,0 @@ -import torch - -def init_weights(m, mean=0.0, std=0.01): - classname = m.__class__.__name__ - if classname.find("Conv") != -1: m.weight.data.normal_(mean, std) - -def get_padding(kernel_size, dilation=1): - return int((kernel_size * dilation - dilation) / 2) - -def convert_pad_shape(pad_shape): - l = pad_shape[::-1] - pad_shape = [item for sublist in l for item in sublist] - return pad_shape - -def slice_segments(x, ids_str, segment_size = 4, dim = 2): - if dim == 2: ret = torch.zeros_like(x[:, :segment_size]) - elif dim == 3: ret = torch.zeros_like(x[:, :, :segment_size]) - - for i in range(x.size(0)): - idx_str = ids_str[i].item() - idx_end = idx_str + segment_size - - if dim == 2: ret[i] = x[i, idx_str:idx_end] - else: ret[i] = x[i, :, idx_str:idx_end] - - return ret - -def rand_slice_segments(x, x_lengths=None, segment_size=4): - b, _, t = x.size() - if x_lengths is None: x_lengths = t - - ids_str_max = x_lengths - segment_size + 1 - ids_str = (torch.rand([b], device=x.device) * ids_str_max).to(dtype=torch.long) - ret = slice_segments(x, ids_str, segment_size, dim=3) - - return ret, ids_str - -@torch.jit.script -def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels): - n_channels_int = n_channels[0] - in_act = input_a + input_b - - t_act = in_act[:, :n_channels_int, :].tanh() - s_act = in_act[:, n_channels_int:, :].sigmoid() - - acts = t_act * s_act - return acts - -def sequence_mask(length, max_length = None): - if max_length is None: max_length = length.max() - return torch.arange(max_length, dtype=length.dtype, device=length.device).unsqueeze(0) < length.unsqueeze(1) - -def clip_grad_value(parameters, clip_value, norm_type=2): - if isinstance(parameters, torch.Tensor): parameters = [parameters] - norm_type = float(norm_type) - - if clip_value is not None: clip_value = float(clip_value) - total_norm = 0 - - for p in list(filter(lambda p: p.grad is not None, parameters)): - total_norm += (p.grad.data.norm(norm_type)).item() ** norm_type - - if clip_value is not None: p.grad.data.clamp_(min=-clip_value, max=clip_value) - - return total_norm ** (1.0 / norm_type) - -def grad_norm(parameters, norm_type = 2.0): - if isinstance(parameters, torch.Tensor): parameters = [parameters] - parameters = [p for p in parameters if p.grad is not None] - - if not parameters: return 0.0 - return torch.linalg.vector_norm(torch.stack([p.grad.norm(norm_type) for p in parameters]), ord=norm_type).item() \ No newline at end of file diff --git a/main/library/algorithm/discriminators.py b/main/library/algorithm/discriminators.py deleted file mode 100644 index a4de94fdda4a17e1d5f5d3091f03b928a3335891..0000000000000000000000000000000000000000 --- a/main/library/algorithm/discriminators.py +++ /dev/null @@ -1,165 +0,0 @@ -import os -import sys -import torch -import torch.nn.functional as F - -from torch.utils.checkpoint import checkpoint -from torch.nn.utils.parametrizations import spectral_norm, weight_norm - -sys.path.append(os.getcwd()) - -from main.library.algorithm.commons import get_padding -from main.library.algorithm.residuals import LRELU_SLOPE - -class MultiPeriodDiscriminator(torch.nn.Module): - def __init__(self, version, use_spectral_norm=False, checkpointing=False): - super(MultiPeriodDiscriminator, self).__init__() - self.checkpointing = checkpointing - - if version == "v0": - periods = [2, 3, 5, 7, 11] - resolutions = [] - elif version == "v1": - periods = [2, 3, 5, 7, 11, 17] - resolutions = [] - elif version == "v2": - periods = [2, 3, 5, 7, 11, 17, 23, 37] - resolutions = [] - elif version == "v3": - periods = [2, 3, 5, 7, 11] - resolutions = [[1024, 120, 600], [2048, 240, 1200], [512, 50, 240]] - - self.discriminators = torch.nn.ModuleList( - [DiscriminatorS(use_spectral_norm=use_spectral_norm)] + - [DiscriminatorP(p, use_spectral_norm=use_spectral_norm) for p in periods] + - [DiscriminatorR(r, use_spectral_norm=use_spectral_norm) for r in resolutions] - ) - - def forward(self, y, y_hat): - y_d_rs, y_d_gs, fmap_rs, fmap_gs = [], [], [], [] - - for d in self.discriminators: - if self.training and self.checkpointing: - y_d_r, fmap_r = checkpoint(d, y, use_reentrant=False) - y_d_g, fmap_g = checkpoint(d, y_hat, use_reentrant=False) - else: - y_d_r, fmap_r = d(y) - y_d_g, fmap_g = d(y_hat) - - y_d_rs.append(y_d_r); fmap_rs.append(fmap_r) - y_d_gs.append(y_d_g); fmap_gs.append(fmap_g) - - return y_d_rs, y_d_gs, fmap_rs, fmap_gs - -class DiscriminatorS(torch.nn.Module): - def __init__(self, use_spectral_norm=False): - super(DiscriminatorS, self).__init__() - norm_f = spectral_norm if use_spectral_norm else weight_norm - self.convs = torch.nn.ModuleList([ - norm_f(torch.nn.Conv1d(1, 16, 15, 1, padding=7)), - norm_f(torch.nn.Conv1d(16, 64, 41, 4, groups=4, padding=20)), - norm_f(torch.nn.Conv1d(64, 256, 41, 4, groups=16, padding=20)), - norm_f(torch.nn.Conv1d(256, 1024, 41, 4, groups=64, padding=20)), - norm_f(torch.nn.Conv1d(1024, 1024, 41, 4, groups=256, padding=20)), - norm_f(torch.nn.Conv1d(1024, 1024, 5, 1, padding=2)) - ]) - self.conv_post = norm_f(torch.nn.Conv1d(1024, 1, 3, 1, padding=1)) - self.lrelu = torch.nn.LeakyReLU(LRELU_SLOPE) - - def forward(self, x): - fmap = [] - for conv in self.convs: - x = self.lrelu(conv(x)) - fmap.append(x) - - x = self.conv_post(x) - fmap.append(x) - - return x.flatten(1, -1), fmap - -class DiscriminatorP(torch.nn.Module): - def __init__(self, period, kernel_size=5, use_spectral_norm=False): - super(DiscriminatorP, self).__init__() - self.period = period - norm_f = spectral_norm if use_spectral_norm else weight_norm - self.convs = torch.nn.ModuleList([ - norm_f( - torch.nn.Conv2d( - in_ch, - out_ch, - (kernel_size, 1), - (stride, 1), - padding=(get_padding(kernel_size, 1), 0) - ) - ) - for in_ch, out_ch, stride in zip( - [1, 32, 128, 512, 1024], - [32, 128, 512, 1024, 1024], - [3, 3, 3, 3, 1] - ) - ]) - self.conv_post = norm_f(torch.nn.Conv2d(1024, 1, (3, 1), 1, padding=(1, 0))) - self.lrelu = torch.nn.LeakyReLU(LRELU_SLOPE) - - def forward(self, x): - fmap = [] - b, c, t = x.shape - if t % self.period != 0: x = F.pad(x, (0, (self.period - (t % self.period))), "reflect") - x = x.view(b, c, -1, self.period) - - for conv in self.convs: - x = self.lrelu(conv(x)) - fmap.append(x) - - x = self.conv_post(x) - fmap.append(x) - return x.flatten(1, -1), fmap - -class DiscriminatorR(torch.nn.Module): - def __init__(self, resolution, use_spectral_norm=False): - super().__init__() - self.resolution = resolution - self.lrelu_slope = 0.1 - norm_f = spectral_norm if use_spectral_norm else weight_norm - self.convs = torch.nn.ModuleList([ - norm_f(torch.nn.Conv2d( 1, 32, (3, 9), padding=(1, 4))), - norm_f(torch.nn.Conv2d(32, 32, (3, 9), stride=(1, 2), padding=(1, 4))), - norm_f(torch.nn.Conv2d(32, 32, (3, 9), stride=(1, 2), padding=(1, 4))), - norm_f(torch.nn.Conv2d(32, 32, (3, 9), stride=(1, 2), padding=(1, 4))), - norm_f(torch.nn.Conv2d(32, 32, (3, 3), padding=(1, 1))) - ]) - self.conv_post = norm_f(torch.nn.Conv2d(32, 1, (3, 3), padding=(1, 1))) - - def forward(self, x): - fmap = [] - x = self.spectrogram(x).unsqueeze(1) - - for layer in self.convs: - x = F.leaky_relu(layer(x), self.lrelu_slope) - fmap.append(x) - - x = self.conv_post(x) - fmap.append(x) - - return x.flatten(1, -1), fmap - - def spectrogram(self, x): - n_fft, hop_length, win_length = self.resolution - pad = int((n_fft - hop_length) / 2) - - is_not_cuda = x.device.type in ["privateuseone", "ocl"] - stft = torch.stft( - F.pad( - x.cpu() if is_not_cuda else x, - (pad, pad), - mode="reflect" - ).squeeze(1), - n_fft=n_fft, - hop_length=hop_length, - win_length=win_length, - window=torch.ones(win_length, device="cpu" if is_not_cuda else x.device), - center=False, - return_complex=True - ) - - return torch.view_as_real(stft).norm(p=2, dim=-1).to(x.device) \ No newline at end of file diff --git a/main/library/algorithm/encoders.py b/main/library/algorithm/encoders.py deleted file mode 100644 index 2b9aab6cc0b2ca118375d758e619ab7f8c4ce922..0000000000000000000000000000000000000000 --- a/main/library/algorithm/encoders.py +++ /dev/null @@ -1,178 +0,0 @@ -import os -import sys -import math -import torch - -sys.path.append(os.getcwd()) - -from main.library.algorithm.modules import WaveNet -from main.library.algorithm.commons import sequence_mask -from main.library.algorithm.normalization import LayerNorm -from main.library.algorithm.attentions import MultiHeadAttention, FFN - -class Encoder(torch.nn.Module): - def __init__( - self, - hidden_channels, - filter_channels, - n_heads, - n_layers, - kernel_size=1, - p_dropout=0.0, - window_size=10, - onnx=False, - **kwargs - ): - super().__init__() - self.hidden_channels = hidden_channels - self.n_layers = n_layers - self.drop = torch.nn.Dropout(p_dropout) - - self.attn_layers = torch.nn.ModuleList([ - MultiHeadAttention( - hidden_channels, - hidden_channels, - n_heads, - p_dropout=p_dropout, - window_size=window_size, - onnx=onnx - ) - for _ in range(n_layers) - ]) - - self.norm_layers_1 = torch.nn.ModuleList([ - LayerNorm( - hidden_channels, - onnx=onnx - ) - for _ in range(n_layers) - ]) - - self.ffn_layers = torch.nn.ModuleList([ - FFN( - hidden_channels, - hidden_channels, - filter_channels, - kernel_size, - p_dropout=p_dropout, - onnx=onnx - ) - for _ in range(n_layers) - ]) - - self.norm_layers_2 = torch.nn.ModuleList([ - LayerNorm( - hidden_channels, - onnx=onnx - ) - for _ in range(n_layers) - ]) - - def forward(self, x, x_mask): - attn_mask = x_mask.unsqueeze(2) * x_mask.unsqueeze(-1) - x = x * x_mask - - for i in range(self.n_layers): - x = self.norm_layers_1[i](x + self.drop(self.attn_layers[i](x, x, attn_mask))) - x = self.norm_layers_2[i](x + self.drop(self.ffn_layers[i](x, x_mask))) - - return x * x_mask - -class TextEncoder(torch.nn.Module): - def __init__( - self, - out_channels, - hidden_channels, - filter_channels, - n_heads, - n_layers, - kernel_size, - p_dropout, - embedding_dim, - f0=True, - energy=False, - onnx=False - ): - super(TextEncoder, self).__init__() - self.hidden_channels = hidden_channels - self.out_channels = out_channels - self.lrelu = torch.nn.LeakyReLU(0.1, inplace=True) - self.emb_phone = torch.nn.Linear(embedding_dim, hidden_channels) - self.emb_pitch = torch.nn.Embedding(256, hidden_channels) if f0 else None - self.emb_energy = torch.nn.Linear(1, hidden_channels) if energy else None - self.encoder = Encoder(hidden_channels, filter_channels, n_heads, n_layers, kernel_size, float(p_dropout), onnx=onnx) - self.proj = torch.nn.Conv1d(hidden_channels, out_channels * 2, 1) - - def forward(self, phone, pitch, lengths, energy): - x = self.emb_phone(phone) - - if pitch is not None: x += self.emb_pitch(pitch) - if energy is not None: x += self.emb_energy(energy.unsqueeze(-1)) - - x = self.lrelu(x * math.sqrt(self.hidden_channels)).transpose(1, -1) - x_mask = sequence_mask(lengths, x.size(2)).unsqueeze(1).to(x.dtype) - m, logs = (self.proj(self.encoder(x * x_mask, x_mask)) * x_mask).split(self.out_channels, dim=1) - - return m, logs, x_mask - -class TextEncoderSVC(torch.nn.Module): - def __init__( - self, - out_channels, - hidden_channels, - filter_channels, - n_heads, - n_layers, - kernel_size, - p_dropout, - onnx=False - ): - super().__init__() - self.hidden_channels = hidden_channels - self.out_channels = out_channels - self.f0_emb = torch.nn.Embedding(256, hidden_channels) - self.proj = torch.nn.Conv1d(hidden_channels, out_channels * 2, 1) - self.encoder = Encoder(hidden_channels, filter_channels, n_heads, n_layers, kernel_size, float(p_dropout), window_size=4, onnx=onnx) - - def forward(self, x, x_mask, f0=None, noise_scale=1): - x = x + self.f0_emb(f0).transpose(1, 2) - - m, logs = (self.proj(self.encoder(x * x_mask, x_mask)) * x_mask).split(self.out_channels, dim=1) - z = (m + torch.randn_like(m) * logs.exp() * noise_scale) * x_mask - - return z, m, logs, x_mask - -class PosteriorEncoder(torch.nn.Module): - def __init__( - self, - in_channels, - out_channels, - hidden_channels, - kernel_size, - dilation_rate, - n_layers, - gin_channels=0 - ): - super(PosteriorEncoder, self).__init__() - self.out_channels = out_channels - self.pre = torch.nn.Conv1d(in_channels, hidden_channels, 1) - self.enc = WaveNet(hidden_channels, kernel_size, dilation_rate, n_layers, gin_channels=gin_channels) - self.proj = torch.nn.Conv1d(hidden_channels, out_channels * 2, 1) - - def forward(self, x, x_lengths, g = None): - x_mask = sequence_mask(x_lengths, x.size(2)).unsqueeze(1).to(x.dtype) - - m, logs = ( - self.proj( - self.enc( - self.pre(x) * x_mask, - x_mask, - g=g - ) - ) * x_mask - ).split(self.out_channels, dim=1) - - return (m + torch.randn_like(m) * logs.exp()) * x_mask, m, logs, x_mask - - def remove_weight_norm(self): - self.enc.remove_weight_norm() \ No newline at end of file diff --git a/main/library/algorithm/faisssearch.py b/main/library/algorithm/faisssearch.py deleted file mode 100644 index e665f4a9f6762f5efd9d22551a0ced1f0be5064d..0000000000000000000000000000000000000000 --- a/main/library/algorithm/faisssearch.py +++ /dev/null @@ -1,67 +0,0 @@ -import os -import sys -import faiss -import torch - -sys.path.append(os.getcwd()) - -from main.app.variables import translations, logger - -class IndexWrapper: - def __init__(self, index_path, nprobe = 1, device="cuda", is_half=False, faiss_cpu=False, clamp=1e-8): - self.index_path = index_path - self.nprobe = nprobe - self.device = device - self.is_half = is_half - self.faiss_cpu = faiss_cpu - self.clamp = clamp - self.index = None - self.big_npy = None - self.b_norms = None - self.big_tensor = None - - def read_index(self): - if self.index_path != "" and os.path.exists(self.index_path): - try: - index = faiss.read_index(self.index_path) - big_npy = index.reconstruct_n(0, index.ntotal) - except Exception as e: - logger.error(translations["read_faiss_index_error"].format(e=e)) - index = big_npy = None - else: index = big_npy = None - - if index is not None: index.nprobe = self.nprobe - self.index = index - self.big_npy = big_npy - - return index, big_npy - - def read_index_tensor(self): - self.read_index() - - if self.index is None or self.big_npy is None: self.big_tensor, self.b_norms = None, None - else: - self.big_tensor = torch.from_numpy(self.big_npy).to(self.device).to(torch.float16 if self.is_half else torch.float32).contiguous() - self.b_norms = (self.big_tensor ** 2).sum(dim=-1, keepdim=True).T.contiguous() - - return self.big_tensor, self.b_norms - - def search(self, query, k=8): - with torch.inference_mode(): - if not self.faiss_cpu: - try: - q_norm = (query ** 2).sum(dim=-1, keepdim=True) - - distances = torch.addmm(self.b_norms, query, self.big_tensor.T, alpha=-2.0, beta=1.0) + q_norm - distances = distances.clamp(min=self.clamp) - - scores, indices = torch.topk(-distances, k=k, dim=-1) - return -scores, indices - except (torch.OutOfMemoryError, RuntimeError): - self.faiss_cpu = True - logger.warning(translations["index_warn"]) - return self.search(query, k) - else: - npy = query.cpu().numpy() - score, ix = self.index.search(npy, k) - return torch.from_numpy(score).to(self.device).to(torch.float16 if self.is_half else torch.float32), torch.from_numpy(ix).to(self.device).long() diff --git a/main/library/algorithm/modules.py b/main/library/algorithm/modules.py deleted file mode 100644 index 53cd0b888384dcad3a6317b10f04314f3a012ff9..0000000000000000000000000000000000000000 --- a/main/library/algorithm/modules.py +++ /dev/null @@ -1,101 +0,0 @@ -import os -import sys -import torch - -import torch.nn.utils.parametrize as parametrize - -sys.path.append(os.getcwd()) - -from .commons import fused_add_tanh_sigmoid_multiply - -class WaveNet(torch.nn.Module): - def __init__( - self, - hidden_channels, - kernel_size, - dilation_rate, - n_layers, - gin_channels=0, - p_dropout=0 - ): - super(WaveNet, self).__init__() - assert kernel_size % 2 == 1 - self.hidden_channels = hidden_channels - self.kernel_size = (kernel_size,) - self.dilation_rate = dilation_rate - self.n_layers = n_layers - self.gin_channels = gin_channels - self.p_dropout = p_dropout - self.n_channels_tensor = torch.IntTensor([hidden_channels]) - self.in_layers = torch.nn.ModuleList() - self.res_skip_layers = torch.nn.ModuleList() - self.drop = torch.nn.Dropout(p_dropout) - - if gin_channels: - self.cond_layer = torch.nn.utils.parametrizations.weight_norm( - torch.nn.Conv1d( - gin_channels, - 2 * hidden_channels * n_layers, - 1 - ), - name="weight" - ) - - dilations = [dilation_rate**i for i in range(n_layers)] - paddings = [(kernel_size * d - d) // 2 for d in dilations] - - for i in range(n_layers): - self.in_layers.append( - torch.nn.utils.parametrizations.weight_norm( - torch.nn.Conv1d( - hidden_channels, - 2 * hidden_channels, - kernel_size, - dilation=dilations[i], - padding=paddings[i] - ), - name="weight" - ) - ) - - res_skip_channels = (hidden_channels if i == n_layers - 1 else 2 * hidden_channels) - - self.res_skip_layers.append( - torch.nn.utils.parametrizations.weight_norm( - torch.nn.Conv1d( - hidden_channels, - res_skip_channels, - 1 - ), - name="weight" - ) - ) - - def forward(self, x, x_mask, g=None): - output = x.clone().zero_() - g = self.cond_layer(g) if g is not None else None - - for i in range(self.n_layers): - x_in = self.in_layers[i](x) - g_l = (g[:, i * 2 * self.hidden_channels : (i + 1) * 2 * self.hidden_channels, :] if g is not None else 0) - res_skip_acts = self.res_skip_layers[i](self.drop(fused_add_tanh_sigmoid_multiply(x_in, g_l, self.n_channels_tensor))) - - if i < self.n_layers - 1: - x = (x + res_skip_acts[:, : self.hidden_channels, :]) * x_mask - output = output + res_skip_acts[:, self.hidden_channels :, :] - else: output = output + res_skip_acts - - return output * x_mask - - def remove_weight_norm(self): - if self.gin_channels != 0: - if hasattr(self.cond_layer, "parametrizations") and "weight" in self.cond_layer.parametrizations: parametrize.remove_parametrizations(self.cond_layer, "weight", leave_parametrized=True) - else: torch.nn.utils.remove_weight_norm(self.cond_layer) - - for l in self.in_layers: - if hasattr(l, "parametrizations") and "weight" in l.parametrizations: parametrize.remove_parametrizations(l, "weight", leave_parametrized=True) - else: torch.nn.utils.remove_weight_norm(l) - - for l in self.res_skip_layers: - if hasattr(l, "parametrizations") and "weight" in l.parametrizations: parametrize.remove_parametrizations(l, "weight", leave_parametrized=True) - else: torch.nn.utils.remove_weight_norm(l) \ No newline at end of file diff --git a/main/library/algorithm/normalization.py b/main/library/algorithm/normalization.py deleted file mode 100644 index 256401683fff3b70c0dcff7a777f0a0503b02aee..0000000000000000000000000000000000000000 --- a/main/library/algorithm/normalization.py +++ /dev/null @@ -1,34 +0,0 @@ -import torch - -import torch.nn.functional as F - -class LayerNorm(torch.nn.Module): - def __init__(self, channels, eps=1e-5, onnx=False): - super().__init__() - self.channels = channels - self.eps = eps - self.gamma = torch.nn.Parameter(torch.ones(channels)) - self.beta = torch.nn.Parameter(torch.zeros(channels)) - self.forward = self._forward_onnx if onnx else self._forward_torch - - def _forward_onnx(self, x): - x = x.transpose(1, -1) - - return F.layer_norm( - x, - (self.channels,), - self.gamma, - self.beta, - self.eps - ).transpose(1, -1) - - def _forward_torch(self, x): - x = x.transpose(1, -1) - - return F.layer_norm( - x, - (x.size(-1),), - self.gamma, - self.beta, - self.eps - ).transpose(1, -1) \ No newline at end of file diff --git a/main/library/algorithm/residuals.py b/main/library/algorithm/residuals.py deleted file mode 100644 index c7da5677a35621c19c7bb002b9e1421e7727a8a8..0000000000000000000000000000000000000000 --- a/main/library/algorithm/residuals.py +++ /dev/null @@ -1,175 +0,0 @@ -import os -import sys -import torch - -import torch.nn.utils.parametrize as parametrize - -from itertools import chain -from torch.nn.utils import remove_weight_norm -from torch.nn.utils.parametrizations import weight_norm - -sys.path.append(os.getcwd()) - -from .modules import WaveNet -from .commons import get_padding, init_weights - -LRELU_SLOPE = 0.1 - -def create_conv1d_layer(channels, kernel_size, dilation): - return weight_norm( - torch.nn.Conv1d( - channels, - channels, - kernel_size, - 1, - dilation=dilation, - padding=get_padding(kernel_size, dilation) - ) - ) - -def apply_mask(tensor, mask): - return tensor * mask if mask is not None else tensor - -class ResBlock(torch.nn.Module): - def __init__( - self, - channels, - kernel_size=3, - dilations=(1, 3, 5) - ): - super().__init__() - self.convs1 = self._create_convs(channels, kernel_size, dilations) - self.convs2 = self._create_convs(channels, kernel_size, [1] * len(dilations)) - - @staticmethod - def _create_convs(channels, kernel_size, dilations): - layers = torch.nn.ModuleList([ - create_conv1d_layer(channels, kernel_size, d) - for d in dilations - ]) - layers.apply(init_weights) - - return layers - - def forward(self, x): - for conv1, conv2 in zip(self.convs1, self.convs2): - y = conv1( - torch.nn.functional.leaky_relu(x, LRELU_SLOPE), - ) - - x = conv2( - torch.nn.functional.leaky_relu(y, LRELU_SLOPE), - ) + x - - return x - - def remove_weight_norm(self): - for conv in chain(self.convs1, self.convs2): - if hasattr(conv, "parametrizations") and "weight" in conv.parametrizations: parametrize.remove_parametrizations(conv, "weight", leave_parametrized=True) - else: remove_weight_norm(conv) - -class Flip(torch.nn.Module): - def forward( - self, - x, - *args, - reverse=False, - **kwargs - ): - x = x.flip([1]) - - if not reverse: return x, torch.zeros(x.size(0)).to(dtype=x.dtype, device=x.device) - else: return x - -class ResidualCouplingBlock(torch.nn.Module): - def __init__( - self, - channels, - hidden_channels, - kernel_size, - dilation_rate, - n_layers, - n_flows=4, - gin_channels=0 - ): - super(ResidualCouplingBlock, self).__init__() - self.channels = channels - self.hidden_channels = hidden_channels - self.kernel_size = kernel_size - self.dilation_rate = dilation_rate - self.n_layers = n_layers - self.n_flows = n_flows - self.gin_channels = gin_channels - self.flows = torch.nn.ModuleList() - - for _ in range(n_flows): - self.flows.append( - ResidualCouplingLayer( - channels, - hidden_channels, - kernel_size, - dilation_rate, - n_layers, - gin_channels=gin_channels, - mean_only=True - ) - ) - self.flows.append(Flip()) - - def forward(self, x, x_mask, g = None, reverse = False): - if not reverse: - for flow in self.flows: - x, _ = flow(x, x_mask, g=g, reverse=reverse) - else: - for flow in reversed(self.flows): - x = flow.forward(x, x_mask, g=g, reverse=reverse) - - return x - - def remove_weight_norm(self): - for i in range(self.n_flows): - self.flows[i * 2].remove_weight_norm() - -class ResidualCouplingLayer(torch.nn.Module): - def __init__( - self, - channels, - hidden_channels, - kernel_size, - dilation_rate, - n_layers, - p_dropout=0, - gin_channels=0, - mean_only=False - ): - assert channels % 2 == 0 - super().__init__() - self.channels = channels - self.hidden_channels = hidden_channels - self.kernel_size = kernel_size - self.dilation_rate = dilation_rate - self.n_layers = n_layers - self.half_channels = channels // 2 - self.mean_only = mean_only - - self.pre = torch.nn.Conv1d(self.half_channels, hidden_channels, 1) - self.enc = WaveNet(hidden_channels, kernel_size, dilation_rate, n_layers, p_dropout=p_dropout, gin_channels=gin_channels) - self.post = torch.nn.Conv1d(hidden_channels, self.half_channels * (2 - mean_only), 1) - - self.post.weight.data.zero_() - self.post.bias.data.zero_() - - def forward(self, x, x_mask, g=None, reverse=False): - x0, x1 = x.split([self.half_channels] * 2, 1) - stats = self.post(self.enc((self.pre(x0) * x_mask), x_mask, g=g)) * x_mask - - if not self.mean_only: m, logs = stats.split([self.half_channels] * 2, 1) - else: - m = stats - logs = torch.zeros_like(m) - - if not reverse: return torch.cat([x0, (m + x1 * logs.exp() * x_mask)], 1), logs.sum(dim=[1, 2]) - else: return torch.cat([x0, ((x1 - m) * (-logs).exp() * x_mask)], 1) - - def remove_weight_norm(self): - self.enc.remove_weight_norm() \ No newline at end of file diff --git a/main/library/algorithm/stftpitchshift.py b/main/library/algorithm/stftpitchshift.py deleted file mode 100644 index 2feb78dd0c901465bc1d7ed86dc81b78ba837e64..0000000000000000000000000000000000000000 --- a/main/library/algorithm/stftpitchshift.py +++ /dev/null @@ -1,270 +0,0 @@ -import numpy as np - -from numpy.lib.stride_tricks import sliding_window_view - -def istft(frames, framesize, hopsize): - frames = np.atleast_2d(frames) - assert frames.ndim == 2 - - analysis_window_size = np.ravel(framesize)[0] - synthesis_window_size = np.ravel(framesize)[-1] - - assert analysis_window_size >= synthesis_window_size - - A = asymmetric_analysis_window(analysis_window_size, synthesis_window_size) if analysis_window_size != synthesis_window_size else symmetric_window(analysis_window_size) - S = asymmetric_synthesis_window(analysis_window_size, synthesis_window_size) if analysis_window_size != synthesis_window_size else symmetric_window(synthesis_window_size) - - W = S * hopsize / np.sum(A * S) - N = frames.shape[0] * hopsize + analysis_window_size - - y = np.zeros((N), float) - - frames[:, 0] = 0 - frames[:, -1] = 0 - frames0 = sliding_window_view(y, analysis_window_size, writeable=True)[::hopsize] - frames1 = np.fft.irfft(frames, axis=-1, norm='forward') * W - - for i in range(min(len(frames0), len(frames1))): - frames0[i] += frames1[i] - - return y - -def asymmetric_synthesis_window(analysis_window_size, synthesis_window_size): - n = analysis_window_size - m = synthesis_window_size // 2 - - right = symmetric_window(2 * m) - window = np.zeros(n) - - window[n-m-m:n-m] = np.square(right[:m]) / symmetric_window(2 * n - 2 * m)[n-m-m:n-m] - window[-m:] = right[-m:] - - return window - -def asymmetric_analysis_window(analysis_window_size, synthesis_window_size): - n = analysis_window_size - m = synthesis_window_size // 2 - - window = np.zeros(n) - window[:n-m] = symmetric_window(2 * n - 2 * m)[:n-m] - window[-m:] = symmetric_window(2 * m)[-m:] - - return window - -def symmetric_window(symmetric_window_size): - n = symmetric_window_size - window = 0.5 - 0.5 * np.cos(2 * np.pi * np.arange(n) / n) - - return window - -def stft(x, framesize, hopsize): - x = np.atleast_1d(x) - assert x.ndim == 1 - - analysis_window_size = np.ravel(framesize)[0] - synthesis_window_size = np.ravel(framesize)[-1] - - assert analysis_window_size >= synthesis_window_size - - W = asymmetric_analysis_window(analysis_window_size, synthesis_window_size) if analysis_window_size != synthesis_window_size else symmetric_window(analysis_window_size) - - frames0 = sliding_window_view(x, analysis_window_size, writeable=False)[::hopsize] - frames1 = np.fft.rfft(frames0 * W, axis=-1, norm='forward') - - return frames1 - -def normalize(frames, frames0): - for i in range(len(frames)): - a = np.real(frames0[i]) - b = np.real(frames[i]) - a = np.dot(a, a) - b = np.dot(b, b) - - if b == 0: continue - frames[i] = np.real(frames[i]) * np.sqrt(a / b) + 1j * np.imag(frames[i]) - - return frames - -def lowpass(cepstrum, quefrency): - cepstrum[1:quefrency] *= 2 - cepstrum[quefrency+1:] = 0 - - return cepstrum - -def lifter(frames, quefrency): - envelopes = np.zeros(frames.shape) - - for i, frame in enumerate(frames): - with np.errstate(divide='ignore', invalid='ignore'): - spectrum = np.log10(np.real(frame)) - - envelopes[i] = np.power(10, np.real(np.fft.rfft(lowpass(np.fft.irfft(spectrum, norm='forward'), quefrency), norm='forward'))) - - return envelopes - -def resample(x, factor): - if factor == 1: return x.copy() - y = np.zeros(x.shape, dtype=x.dtype) - - n = len(x) - m = int(n * factor) - - i = np.arange(min(n, m)) - k = i * (n / m) - - j = np.trunc(k).astype(int) - k = k - j - - ok = (0 <= j) & (j < n - 1) - y[i[ok]] = k[ok] * x[j[ok] + 1] + (1 - k[ok]) * x[j[ok]] - - return y - -def shiftpitch(frames, factors, samplerate): - for i in range(len(frames)): - magnitudes = np.vstack([resample(np.real(frames[i]), factor) for factor in factors]) - frequencies = np.vstack([resample(np.imag(frames[i]), factor) * factor for factor in factors]) - - magnitudes[(frequencies <= 0) | (frequencies >= samplerate / 2)] = 0 - mask = np.argmax(magnitudes, axis=0) - - magnitudes = np.take_along_axis(magnitudes, mask[None,:], axis=0) - frequencies = np.take_along_axis(frequencies, mask[None,:], axis=0) - - frames[i] = magnitudes + 1j * frequencies - - return frames - -def wrap(x): - return (x + np.pi) % (2 * np.pi) - np.pi - -def encode(frames, framesize, hopsize, samplerate): - M, N = frames.shape - analysis_framesize = np.ravel(framesize)[0] - - freqinc = samplerate / analysis_framesize - phaseinc = 2 * np.pi * hopsize / analysis_framesize - - buffer = np.zeros(N) - data = np.zeros((M, N), complex) - - for m, frame in enumerate(frames): - arg = np.angle(frame) - delta = arg - buffer - - buffer = arg - - i = np.arange(N) - data[m] = np.abs(frame) + 1j * ((i + (wrap(delta - i * phaseinc) / phaseinc)) * freqinc) - - return data - -def decode(frames, framesize, hopsize, samplerate): - M, N = frames.shape - analysis_framesize = np.ravel(framesize)[0] - synthesis_framesize = np.ravel(framesize)[-1] - - freqinc = samplerate / analysis_framesize - phaseinc = 2 * np.pi * hopsize / analysis_framesize - timeshift = 2 * np.pi * synthesis_framesize * np.arange(N) / N if synthesis_framesize != analysis_framesize else 0 - - buffer = np.zeros(N) - data = np.zeros((M, N), complex) - - for m, frame in enumerate(frames): - i = np.arange(N) - delta = (i + ((np.imag(frame) - i * freqinc) / freqinc)) * phaseinc - buffer += delta - arg = buffer.copy() - arg -= timeshift - data[m] = np.real(frame) * np.exp(1j * arg) - - return data - -class StftPitchShift: - def __init__(self, framesize, hopsize, samplerate): - self.framesize = framesize - self.hopsize = hopsize - self.samplerate = samplerate - - def shiftpitch(self, input, factors = 1, quefrency = 0, distortion = 1, normalization = False): - input = np.atleast_1d(input) - dtype = input.dtype - shape = input.shape - - input = np.squeeze(input) - if input.ndim != 1: raise ValueError - - if np.issubdtype(dtype, np.integer): - a, b = np.iinfo(dtype).min, np.iinfo(dtype).max - - input = ((input.astype(float) - a) / (b - a)) * 2 - 1 - elif not np.issubdtype(dtype, np.floating): raise TypeError - - def isnotnormal(x): - return (np.isinf(x)) | (np.isnan(x)) | (abs(x) < np.finfo(x.dtype).tiny) - - framesize = self.framesize - hopsize = self.hopsize - samplerate = self.samplerate - - factors = np.asarray(factors).flatten() - quefrency = int(quefrency * samplerate) - - frames = encode( - stft( - input, - framesize, - hopsize - ), - framesize, - hopsize, - samplerate - ) - - if normalization: frames0 = frames.copy() - - if quefrency: - envelopes = lifter(frames, quefrency) - mask = isnotnormal(envelopes) - - frames.real /= envelopes - frames.real[mask] = 0 - - if distortion != 1: - envelopes[mask] = 0 - - for i in range(len(envelopes)): - envelopes[i] = resample(envelopes[i], distortion) - - mask = isnotnormal(envelopes) - - frames = shiftpitch( - frames, - factors, - samplerate - ) - - frames.real *= envelopes - frames.real[mask] = 0 - else: frames = shiftpitch(frames, factors, samplerate) - - if normalization: - frames = normalize( - frames, - frames0 - ) - - output = istft(decode(frames, framesize, hopsize, samplerate), framesize, hopsize) - output.resize(shape, refcheck=False) - - if np.issubdtype(dtype, np.integer): - a, b = np.iinfo(dtype).min, np.iinfo(dtype).max - - output = (((output + 1) / 2) * (b - a) + a).clip(a, b).astype(dtype) - elif output.dtype != dtype: output = output.astype(dtype) - - assert output.dtype == dtype - assert output.shape == shape - - return output \ No newline at end of file diff --git a/main/library/algorithm/synthesizers.py b/main/library/algorithm/synthesizers.py deleted file mode 100644 index 01896efa042206031b5f4b59472ce15b08bb53b2..0000000000000000000000000000000000000000 --- a/main/library/algorithm/synthesizers.py +++ /dev/null @@ -1,394 +0,0 @@ -import os -import sys -import torch - -sys.path.append(os.getcwd()) - -from main.app.variables import logger, translations -from main.library.algorithm.residuals import ResidualCouplingBlock -from main.library.algorithm.encoders import TextEncoder, PosteriorEncoder, TextEncoderSVC -from main.library.algorithm.commons import slice_segments, rand_slice_segments, sequence_mask - -class Synthesizer(torch.nn.Module): - def __init__( - self, - spec_channels, - segment_size, - inter_channels, - hidden_channels, - filter_channels, - n_heads, - n_layers, - kernel_size, - p_dropout, - resblock, - resblock_kernel_sizes, - resblock_dilation_sizes, - upsample_rates, - upsample_initial_channel, - upsample_kernel_sizes, - spk_embed_dim, - gin_channels, - sr, - use_f0, - text_enc_hidden_dim=768, - vocoder="Default", - checkpointing=False, - onnx=False, - energy=False, - **kwargs - ): - super(Synthesizer, self).__init__() - self.segment_size = segment_size - self.use_f0 = use_f0 - self.enc_p = TextEncoder( - inter_channels, - hidden_channels, - filter_channels, - n_heads, - n_layers, - kernel_size, - float(p_dropout), - text_enc_hidden_dim, - f0=use_f0, - energy=energy, - onnx=onnx - ) - - if use_f0: - if vocoder == "RefineGAN": - from main.library.generators.refinegan import RefineGANGenerator - - logger.info(translations["use_vocoders"].format(name="REFINEGAN")) - - self.dec = RefineGANGenerator( - sample_rate=sr, - upsample_rates=upsample_rates, - num_mels=inter_channels, - checkpointing=checkpointing - ) - elif vocoder == "BigVGAN": - from main.library.generators.bigvgan import BigVGANGenerator - - logger.info(translations["use_vocoders"].format(name="BIGVGAN")) - - self.dec = BigVGANGenerator( - in_channel=inter_channels, - upsample_initial_channel=upsample_initial_channel, - upsample_rates=upsample_rates, - upsample_kernel_sizes=upsample_kernel_sizes, - resblock_kernel_sizes=resblock_kernel_sizes, - resblock_dilations=resblock_dilation_sizes, - gin_channels=gin_channels, - sample_rate=sr, - harmonic_num=0, - ) - elif vocoder in ["MRF-HiFi-GAN", "MRF HiFi-GAN"]: - from main.library.generators.mrf_hifigan import HiFiGANMRFGenerator - - logger.info(translations["use_vocoders"].format(name="MRF-HIFIGAN")) - - self.dec = HiFiGANMRFGenerator( - in_channel=inter_channels, - upsample_initial_channel=upsample_initial_channel, - upsample_rates=upsample_rates, - upsample_kernel_sizes=upsample_kernel_sizes, - resblock_kernel_sizes=resblock_kernel_sizes, - resblock_dilations=resblock_dilation_sizes, - gin_channels=gin_channels, - sample_rate=sr, - harmonic_num=8, - checkpointing=checkpointing - ) - else: - from main.library.generators.nsf_hifigan import HiFiGANNSFGenerator - - logger.info(translations["use_vocoders"].format(name="NSF-HIFIGAN")) - - self.dec = HiFiGANNSFGenerator( - inter_channels, - resblock_kernel_sizes, - resblock_dilation_sizes, - upsample_rates, - upsample_initial_channel, - upsample_kernel_sizes, - gin_channels=gin_channels, - sr=sr, - checkpointing=checkpointing, - harmonic_num=0 - ) - else: - from main.library.generators.hifigan import HiFiGANGenerator - - logger.info(translations["use_vocoders"].format(name="HIFIGAN")) - - self.dec = HiFiGANGenerator( - inter_channels, - resblock_kernel_sizes, - resblock_dilation_sizes, - upsample_rates, - upsample_initial_channel, - upsample_kernel_sizes, - gin_channels=gin_channels - ) - - self.enc_q = PosteriorEncoder(spec_channels, inter_channels, hidden_channels, 5, 1, 16, gin_channels=gin_channels) - self.flow = ResidualCouplingBlock(inter_channels, hidden_channels, 5, 1, 3, gin_channels=gin_channels) - self.emb_g = torch.nn.Embedding(spk_embed_dim, gin_channels) - - def remove_weight_norm(self): - for module in [self.dec, self.flow, self.enc_q]: - module.remove_weight_norm() - - @torch.jit.ignore - def forward(self, phone, phone_lengths, pitch = None, pitchf = None, y = None, y_lengths = None, ds = None, energy = None): - g = self.emb_g(ds).unsqueeze(-1) - m_p, logs_p, x_mask = self.enc_p(phone, pitch, phone_lengths, energy) - - if y is not None: - z, m_q, logs_q, y_mask = self.enc_q(y, y_lengths, g=g) - z_p = self.flow(z, y_mask, g=g) - z_slice, ids_slice = rand_slice_segments(z, y_lengths, self.segment_size) - - return ( - (self.dec( - z_slice, - slice_segments(pitchf, ids_slice, self.segment_size, 2), - g=g - ) if self.use_f0 else self.dec( - z_slice, - g=g - )), - ids_slice, - x_mask, - y_mask, - (z, z_p, m_p, logs_p, m_q, logs_q) - ) - else: return None, None, x_mask, None, (None, None, m_p, logs_p, None, None) - - @torch.jit.export - def infer(self, phone, phone_lengths, pitch = None, nsff0 = None, sid = None, energy = None): - g = self.emb_g(sid).unsqueeze(-1) - m_p, logs_p, x_mask = self.enc_p(phone, pitch, phone_lengths, energy) - z_p = (m_p + logs_p.exp() * torch.randn_like(m_p) * 0.66666) * x_mask - - z = self.flow(z_p, x_mask, g=g, reverse=True) - o = self.dec(z * x_mask, nsff0, g=g) if self.use_f0 else self.dec(z * x_mask, g=g) - - return o, x_mask, (z, z_p, m_p, logs_p) - - def onnx_infer(self, phone, phone_lengths, pitch = None, nsff0 = None, sid = None, energy = None): - g = self.emb_g(sid).unsqueeze(-1) - m_p, logs_p, x_mask = self.enc_p(phone, pitch, phone_lengths, energy) - z_p = (m_p + logs_p.exp() * torch.randn_like(m_p) * 0.66666) * x_mask - z = self.flow(z_p, x_mask, g=g, reverse=True) - - return self.dec( - z * x_mask, - nsff0, - g=g - ) if self.use_f0 else self.dec( - z * x_mask, - g=g - ) - -class SynthesizerSVC(torch.nn.Module): - def __init__( - self, - spec_channels, - segment_size, - inter_channels, - hidden_channels, - filter_channels, - n_heads, - n_layers, - kernel_size, - p_dropout, - resblock, - resblock_kernel_sizes, - resblock_dilation_sizes, - upsample_rates, - upsample_initial_channel, - upsample_kernel_sizes, - spk_embed_dim, - gin_channels, - sr, - text_enc_hidden_dim=768, - vocoder="Default", - checkpointing=False, - onnx=False, - noise_scale=0.35, - **kwargs - ): - - super().__init__() - self.segment_size = segment_size - self.noise_scale = noise_scale - self.sr = sr - - self.enc_p = TextEncoderSVC( - inter_channels, - hidden_channels, - filter_channels=filter_channels, - n_heads=n_heads, - n_layers=n_layers, - kernel_size=kernel_size, - p_dropout=p_dropout, - onnx=onnx - ) - - if vocoder == "RefineGAN": - from main.library.generators.refinegan import RefineGANGenerator - - logger.info(translations["use_vocoders"].format(name="REFINEGAN")) - - self.dec = RefineGANGenerator( - sample_rate=sr, - upsample_rates=upsample_rates, - num_mels=inter_channels, - checkpointing=checkpointing - ) - elif vocoder == "BigVGAN": - from main.library.generators.bigvgan import BigVGANGenerator - - logger.info(translations["use_vocoders"].format(name="BIGVGAN")) - - self.dec = BigVGANGenerator( - in_channel=inter_channels, - upsample_initial_channel=upsample_initial_channel, - upsample_rates=upsample_rates, - upsample_kernel_sizes=upsample_kernel_sizes, - resblock_kernel_sizes=resblock_kernel_sizes, - resblock_dilations=resblock_dilation_sizes, - gin_channels=gin_channels, - sample_rate=sr, - harmonic_num=0, - ) - elif vocoder in ["MRF-HiFi-GAN", "MRF HiFi-GAN"]: - from main.library.generators.mrf_hifigan import HiFiGANMRFGenerator - - logger.info(translations["use_vocoders"].format(name="MRF-HIFIGAN")) - - self.dec = HiFiGANMRFGenerator( - in_channel=inter_channels, - upsample_initial_channel=upsample_initial_channel, - upsample_rates=upsample_rates, - upsample_kernel_sizes=upsample_kernel_sizes, - resblock_kernel_sizes=resblock_kernel_sizes, - resblock_dilations=resblock_dilation_sizes, - gin_channels=gin_channels, - sample_rate=sr, - harmonic_num=8, - checkpointing=checkpointing - ) - else: - from main.library.generators.nsf_hifigan import HiFiGANNSFGenerator - - logger.info(translations["use_vocoders"].format(name="NSF-HIFIGAN")) - - self.dec = HiFiGANNSFGenerator( - inter_channels, - resblock_kernel_sizes, - resblock_dilation_sizes, - upsample_rates, - upsample_initial_channel, - upsample_kernel_sizes, - gin_channels=gin_channels, - sr=sr, - checkpointing=checkpointing, - harmonic_num=8 - ) - - self.emb_uv = torch.nn.Embedding(2, hidden_channels) - self.emb_g = torch.nn.Embedding(spk_embed_dim, gin_channels) - self.pre = torch.nn.Conv1d(text_enc_hidden_dim, hidden_channels, kernel_size=5, padding=2) - self.enc_q = PosteriorEncoder(spec_channels, inter_channels, hidden_channels, 5, 1, 16, gin_channels=gin_channels) - self.flow = ResidualCouplingBlock(inter_channels, hidden_channels, 5, 1, 4, gin_channels=gin_channels) - - def remove_weight_norm(self): - for module in [self.dec, self.flow, self.enc_q]: - module.remove_weight_norm() - - def forward(self, phone, phone_lengths, pitch = None, pitchf = None, y = None, y_lengths = None, ds = None, energy = None): - g = self.emb_g(ds.unsqueeze(0) if ds.dim() == 1 else ds).transpose(1, 2) - phone = phone.transpose(1, 2) - - x_mask = sequence_mask(phone_lengths, phone.size(2)).unsqueeze(1).to(phone.dtype) - x = self.pre(phone) * x_mask + self.emb_uv((pitchf > 0.0).long()).transpose(1, 2) - - _, m_p, logs_p, _ = self.enc_p(x, x_mask, f0=pitch) - z, m_q, logs_q, spec_mask = self.enc_q(y, y_lengths, g=g) - - z_p = self.flow(z, spec_mask, g=g) - z_slice, ids_slice = rand_slice_segments(z, y_lengths, self.segment_size) - - return ( - self.dec( - z_slice, - g=g, - f0=slice_segments(pitchf, ids_slice, self.segment_size, 2) - ), - ids_slice, - x_mask, - spec_mask, - (z, z_p, m_p, logs_p, m_q, logs_q) - ) - - @torch.no_grad() - def infer(self, phone, phone_lengths, pitch = None, nsff0 = None, sid = None, energy = None): - g = self.emb_g(sid.unsqueeze(0) if sid.dim() == 1 else sid).transpose(1, 2) - phone = phone.transpose(1, 2) - - x_mask = sequence_mask(phone_lengths, phone.size(2)).unsqueeze(1).to(phone.dtype) - x = self.pre(phone) * x_mask + self.emb_uv((nsff0 > 0.0).long()).transpose(1, 2) - - z_p, m_p, logs_p, c_mask = self.enc_p(x, x_mask, f0=pitch, noise_scale=self.noise_scale) - z = self.flow(z_p, c_mask, g=g, reverse=True) - o = self.dec(z * c_mask, g=g, f0=nsff0) - - target_len = phone_lengths * (self.sr // 100) - if o.shape[-1] != target_len: - o = torch.nn.functional.interpolate( - o, - size=target_len, - mode="linear", - align_corners=False - ) - - return o, x_mask, (z, z_p, m_p, logs_p) - - @torch.no_grad() - def onnx_infer(self, phone, phone_lengths, pitch = None, nsff0 = None, sid = None, energy = None): - g = self.emb_g(sid.unsqueeze(0) if sid.dim() == 1 else sid).transpose(1, 2) - phone = phone.transpose(1, 2) - - x_mask = sequence_mask(phone_lengths, phone.size(2)).unsqueeze(1).to(phone.dtype) - - z_p, _, _, c_mask = self.enc_p( - self.pre(phone) * x_mask + self.emb_uv((nsff0 > 0.0).long()).transpose(1, 2), - x_mask, - f0=pitch, - noise_scale=self.noise_scale - ) - - o = self.dec( - self.flow( - z_p, - c_mask, - g=g, - reverse=True - ) * c_mask, - g=g, - f0=nsff0 - ) - - target_len = phone_lengths * (self.sr // 100) - if o.shape[-1] != target_len: - o = torch.nn.functional.interpolate( - o, - size=target_len, - mode="linear", - align_corners=False - ) - - return o \ No newline at end of file diff --git a/main/library/algorithm/viterbi.py b/main/library/algorithm/viterbi.py deleted file mode 100644 index b33c6846835a0f790823c5f0fa4e66e66528404b..0000000000000000000000000000000000000000 --- a/main/library/algorithm/viterbi.py +++ /dev/null @@ -1,130 +0,0 @@ -import torch - -import numpy as np -import numba as nb - -@nb.njit(cache=True) -def _viterbi(log_prob, log_trans_T, log_p_init): - n_steps, n_states = log_prob.shape - - state = np.empty(n_steps, dtype=np.int16) - ptr = np.empty((n_steps, n_states), dtype=np.int16) - - prev = np.empty(n_states, dtype=log_prob.dtype) - curr = np.empty(n_states, dtype=log_prob.dtype) - - for j in range(n_states): - prev[j] = log_prob[0, j] + log_p_init[j] - - for t in range(1, n_steps): - for j in range(n_states): - row = log_trans_T[j] - best_val = prev[0] + row[0] - best_idx = 0 - - for i in range(1, n_states): - v = prev[i] + row[i] - if v > best_val: best_val, best_idx = v, i - - curr[j] = log_prob[t, j] + best_val - ptr[t, j] = best_idx - - prev, curr = curr, prev - - best_idx = 0 - best_val = prev[0] - - for j in range(1, n_states): - v = prev[j] - if v > best_val: best_val, best_idx = v, j - - state[-1] = best_idx - for t in range(n_steps - 2, -1, -1): - state[t] = ptr[t + 1, state[t + 1]] - - return state, best_val - -def viterbi_np(prob, transition, p_init=None, return_logp=False, eps=1e-7): - n_states, _ = prob.shape[-2:] - - if transition.shape != (n_states, n_states): raise ValueError - if np.any(transition < 0) or not np.allclose(transition.sum(axis=1), 1): raise ValueError - if np.any(prob < 0) or np.any(prob > 1): raise ValueError - - if p_init is None: p_init = np.full(n_states, 1.0 / n_states, dtype=prob.dtype) - elif (np.any(p_init < 0) or not np.allclose(p_init.sum(), 1) or p_init.shape != (n_states,)): raise ValueError - - log_trans = np.ascontiguousarray(np.log(transition + eps)).T - log_prob = np.ascontiguousarray(np.log(prob + eps)) - log_p_init = np.ascontiguousarray(np.log(p_init + eps)) - - if log_prob.ndim == 2: states, logp = _viterbi(log_prob.T, log_trans, log_p_init) - else: - batch_shape = log_prob.shape[:-2] - n_steps = log_prob.shape[-1] - - flat = log_prob.reshape(-1, n_states, n_steps) - states = np.empty((flat.shape[0], n_steps), dtype=np.int16) - logp = np.empty(flat.shape[0], dtype=log_prob.dtype) - - for i in range(flat.shape[0]): - s, lp = _viterbi(flat[i].T, log_trans, log_p_init) - states[i] = s - logp[i] = lp - - states = states.reshape(*batch_shape, n_steps) - logp = logp.reshape(batch_shape) - - if return_logp: return states, logp - return states - -def viterbi(prob, transition, p_init=None, return_logp=False, eps=1e-7): - if not prob.device.type.startswith(("cuda", "cpu")): - states_np, logp_np = viterbi_np( - prob.detach().cpu().numpy(), - transition.detach().cpu().numpy(), - p_init.detach().cpu().numpy() if p_init is not None else p_init, - return_logp=True, - eps=eps - ) - states_pt = torch.as_tensor(states_np, device=prob.device, dtype=torch.int32) - - if return_logp: return states_pt, torch.as_tensor(logp_np, device=prob.device) - return states_pt - - n_states = prob.shape[-2] - n_steps = prob.shape[-1] - batch_shape = prob.shape[:-2] - - if transition.shape != (n_states, n_states): raise ValueError - if torch.any(transition < 0) or not torch.allclose(transition.sum(dim=-1), torch.tensor(1.0, dtype=transition.dtype, device=transition.device)): raise ValueError - if torch.any(prob < 0) or torch.any(prob > 1): raise ValueError - - if p_init is None: p_init = torch.full((n_states,), 1.0 / n_states, dtype=prob.dtype, device=prob.device) - elif torch.any(p_init < 0) or not torch.allclose(p_init.sum(), torch.tensor(1.0, dtype=p_init.dtype, device=p_init.device)) or p_init.shape != (n_states,): raise ValueError - - log_trans = transition.clamp_min(eps).log() - log_prob = prob.clamp_min(eps).log() - log_p_init = p_init.clamp_min(eps).log() - - value = torch.zeros((*batch_shape, n_steps, n_states), dtype=prob.dtype, device=prob.device) - ptr = torch.zeros((*batch_shape, n_steps, n_states), dtype=torch.long, device=prob.device) - - value[..., 0, :] = log_prob[..., :, 0] + log_p_init - log_trans_expanded = log_trans.unsqueeze(0) - - for t in range(1, n_steps): - max_vals, argmax_indices = (value[..., t - 1, :].unsqueeze(-1) + log_trans_expanded).max(dim=-2) - - ptr[..., t, :] = argmax_indices - value[..., t, :] = log_prob[..., :, t] + max_vals - - states = torch.zeros((*batch_shape, n_steps), dtype=torch.long, device=prob.device) - states[..., -1] = value[..., -1, :].argmax(dim=-1) - - for t in range(n_steps - 2, -1, -1): - next_state = states[..., t + 1].unsqueeze(-1) - states[..., t] = ptr[..., t + 1, :].gather(dim=-1, index=next_state).squeeze(-1) - - if return_logp: return states.to(torch.int64), value[..., -1, :].gather(dim=-1, index=states[..., -1].unsqueeze(-1)).squeeze(-1) - return states.to(torch.int64) \ No newline at end of file diff --git a/main/library/audio/audio.py b/main/library/audio/audio.py deleted file mode 100644 index 188afed0f3c5b6a1868fd9e2cf0f60697cb7b1a0..0000000000000000000000000000000000000000 --- a/main/library/audio/audio.py +++ /dev/null @@ -1,141 +0,0 @@ -import os -import sys -import librosa - -import numpy as np -import soundfile as sf - -from pydub import AudioSegment - -sys.path.append(os.getcwd()) - -from main.app.variables import translations - -def load_audio( - file, - sample_rate=16000, - formant_shifting=False, - formant_qfrency=0.8, - formant_timbre=0.8, - flatten=True, - res_type="soxr_vhq", - dtype = np.float64, - return_sr = False, - mode = "soundfile" -): - try: - file = file.strip(" ").strip('"').strip("\n").strip('"').strip(" ") - if not os.path.isfile(file): raise FileNotFoundError(translations["not_found"].format(name=file)) - - try: - if mode == "soundfile": audio, sr = sf.read(file, dtype=dtype) - elif mode == "librosa": audio, sr = librosa.load(file, sr=None, dtype=dtype) - elif mode == "ffmpeg": - from main.library.audio import ffmpeg - - audio, _ = ( - ffmpeg.input(file, threads=0) - .output("-", format="f32le", acodec="pcm_f32le", ac=1, ar=sample_rate) - .run(cmd=["ffmpeg", "-nostdin"], capture_stdout=True, capture_stderr=True) - ) - audio = np.frombuffer(audio, dtype=np.float32).astype(dtype=dtype) - sr = sample_rate - else: - raise ValueError - except: - modes = ["soundfile", "librosa", "ffmpeg"] - if mode in modes: modes.remove(mode) - - for m in modes: - try: - return load_audio( - file, - sample_rate=sample_rate, - formant_shifting=formant_shifting, - formant_qfrency=formant_qfrency, - formant_timbre=formant_timbre, - flatten=flatten, - res_type=res_type, - dtype=dtype, - return_sr=return_sr, - mode=m - ) - except: - pass - raise RuntimeError - - if audio.ndim > 1: audio = audio.mean(axis=0, dtype=dtype) if audio.shape[0] < audio.shape[1] else audio.mean(axis=1, dtype=dtype) - - if sr is not None and sr != sample_rate: - audio = librosa.resample( - audio, - orig_sr=sr, - target_sr=sample_rate, - res_type=res_type - ) - sr = sample_rate - - if formant_shifting: - from main.library.algorithm.stftpitchshift import StftPitchShift - - pitchshifter = StftPitchShift( - framesize=1024, - hopsize=32, - samplerate=sample_rate - ) - - audio = pitchshifter.shiftpitch( - audio, - factors=1, - quefrency=formant_qfrency * 1e-3, - distortion=formant_timbre - ) - - audio = audio.flatten() if flatten else audio - except Exception as e: - raise RuntimeError(f"{translations['errors_loading_audio']}: {e}") - - if return_sr: return audio, sr - return audio - -def pydub_load(input_path, volume = None): - try: - if input_path.endswith(".wav"): audio = AudioSegment.from_wav(input_path) - elif input_path.endswith(".mp3"): audio = AudioSegment.from_mp3(input_path) - elif input_path.endswith(".ogg"): audio = AudioSegment.from_ogg(input_path) - else: audio = AudioSegment.from_file(input_path) - except: - audio = AudioSegment.from_file(input_path) - - return audio if volume is None else (audio + volume) - -def cut(audio, sr, db_thresh=-60, min_interval=250): - from main.inference.preprocess.slicer2 import Slicer2 - - slicer = Slicer2( - sr=sr, - threshold=db_thresh, - min_interval=min_interval - ) - - return slicer.slice2(audio) - -def restore(segments, total_len, dtype=np.float32): - out = [] - last_end = 0 - - for start, end, processed_seg in segments: - if start > last_end: - out.append( - np.zeros(start - last_end, dtype=dtype) - ) - - out.append(processed_seg) - last_end = end - - if last_end < total_len: - out.append( - np.zeros(total_len - last_end, dtype=dtype) - ) - - return np.concatenate(out, axis=-1) \ No newline at end of file diff --git a/main/library/audio/audio_processing.py b/main/library/audio/audio_processing.py deleted file mode 100644 index 15b3a662bcdec9b4ee8cf6f33407ca599669df87..0000000000000000000000000000000000000000 --- a/main/library/audio/audio_processing.py +++ /dev/null @@ -1,119 +0,0 @@ -import os -import sys -import librosa - -import numpy as np -import scipy.signal as signal - -sys.path.append(os.getcwd()) - -def rms(x, eps=1e-9): - return np.sqrt(np.mean(x**2) + eps) - -def dc_remove(x): - return x - np.mean(x) - -def normalize_rms(x, target=0.08): - r = rms(x) - return x if r < 1e-6 else x * (target / r) - -def soft_limiter(x, th=0.98): - return np.where( - np.abs(x) < th, - x, - th * np.sign(x) + (x - th * np.sign(x)) / (1 + ((x - th) / (1 - th))**2) - ) - -def preprocess( - audio, - target_sr=16000 -): - x = dc_remove(audio) - x = signal.filtfilt(*signal.butter(2, 80 / (target_sr / 2), btype="high"), x) - - x = normalize_rms(x, 0.06) - x = spectral_denoise_np(x, alpha=1.0) - - x /= (np.max(np.abs(x)) + 1e-9) * 0.99 - return x.astype(np.float32) - -def spectral_denoise_np( - audio, - n_fft=4096, - hop=1024, - alpha=2.5 -): - if hop is None: - hop = n_fft // 4 - - stft = librosa.stft( - audio, - n_fft=n_fft, - hop_length=hop, - window="hann" - ) - - mag = np.abs(stft) - phase = np.angle(stft) - - energy = np.mean(mag**2, axis=0) - silent = energy < np.percentile(energy, 30) - - mask = (mag ** 2) / (mag ** 2 + alpha * ((np.median(mag[:, silent], axis=1, keepdims=True) if np.any(silent) else np.median(mag, axis=1, keepdims=True)) ** 2) + 1e-9) - clean = mask * mag - - out = librosa.istft( - clean * np.exp(1j * phase), - hop_length=hop, - length=len(audio) - ) - - return out.astype(np.float32) - -def output_eq(audio, sr): - out = audio.copy() - - bands = [ - (0, 60, -8.0), - (60, 200, -3.0), - (200, 1000, 0.0), - (1000, 3500, +2.5), - (3500, 8000, +3.0), - (8000, 16000, +1.5) - ] - - for low, high, g in bands: - gain = 10 ** (g / 20) - - if low == 0: - b, a = signal.butter(2, high / (sr / 2), "low") - else: - b, a = signal.butter(2, [low / (sr / 2), high / (sr / 2)], "band") - - band = signal.filtfilt(b, a, audio) - out += (gain - 1.0) * band - - return out - -def presence_exciter(audio, sr, strength=0.03): - b, a = signal.butter(2, [4000 / (sr / 2), 9000 / (sr / 2)], "band") - band = signal.filtfilt(b, a, audio) - - exc = np.sign(band) * (1 - np.exp(-np.abs(band) * 4.0)) - return audio + strength * exc - -def postprocess( - audio, - sr=48000 -): - x = dc_remove(audio) - x = signal.filtfilt(*signal.butter(2, 60 / (sr / 2), "high"), x) - - x = spectral_denoise_np(x) - x = output_eq(x, sr) - - x = presence_exciter(x, sr) - x = soft_limiter(x, 0.98) - - x /= (np.max(np.abs(x)) + 1e-9) * 0.99 - return x.astype(np.float32) \ No newline at end of file diff --git a/main/library/audio/features.py b/main/library/audio/features.py deleted file mode 100644 index cc8918e0b56f86094a13d149487ba7e1c5c6d744..0000000000000000000000000000000000000000 --- a/main/library/audio/features.py +++ /dev/null @@ -1,146 +0,0 @@ -import torch - -import torch.nn.functional as F - -def frame(x, frame_length, hop_length, axis = -1): - if x.shape[axis] < frame_length or hop_length < 1: raise ValueError - - axis = axis % x.ndim - if axis != x.ndim - 1: x = x.movedim(axis, -1) - - xw = torch.as_strided(x, size=x.shape[:-1] + (1 + (x.shape[-1] - frame_length) // hop_length, frame_length), stride=x.stride()[:-1] + (hop_length * x.stride()[-1], x.stride()[-1])) - if axis != x.ndim - 1: xw = xw.movedim(-2, axis) - - return xw - -def rms( - y, - frame_length = 2048, - hop_length = 512, - center = True, - pad_mode = "constant", - dtype = torch.float32, - device = "cpu" -): - y = y.to(device=device, dtype=dtype) if torch.is_tensor(y) else torch.from_numpy(y.copy()).to(device=device, dtype=dtype) - return frame(torch.nn.functional.pad(y, (frame_length // 2, frame_length // 2), mode=pad_mode) if center else y, frame_length=frame_length, hop_length=hop_length).square().mean(dim=-1, keepdim=True).sqrt().mT - -def change_rms(source_audio, source_rate, target_audio, target_rate, rate, device): - if not torch.is_tensor(target_audio): target_audio = torch.from_numpy(target_audio).to(device=device, dtype=torch.float32) - - rms2 = F.interpolate( - rms( - y=target_audio, - frame_length=target_rate // 2 * 2, - hop_length=target_rate // 2, - device=device - ).float().unsqueeze(0), - size=target_audio.shape[0], - mode="linear" - ).squeeze() - - return target_audio * ( - F.interpolate( - rms( - y=source_audio, - frame_length=source_rate // 2 * 2, - hop_length=source_rate // 2, - device=device - ).float().unsqueeze(0), - size=target_audio.shape[0], - mode="linear" - ).squeeze().pow(1 - rate) * rms2.maximum(torch.zeros_like(rms2) + 1e-6).pow(rate - 1) - ) - -def mel(sr, n_fft, n_mels=128, fmin=0.0, fmax=None, htk=False, norm="slaney", dtype=torch.float32, device=None): - if fmax is None: fmax = float(sr) / 2 - n_mels = int(n_mels) - - weights = torch.zeros((n_mels, int(1 + n_fft // 2)), dtype=dtype, device=device) - - fftfreqs = torch.fft.rfftfreq(n_fft, d=1.0 / sr, device=device).to(dtype) - mel_f = mel_to_hz(torch.linspace(hz_to_mel(fmin, htk=htk, dtype=dtype, device=device), hz_to_mel(fmax, htk=htk, dtype=dtype, device=device), n_mels + 2, dtype=dtype, device=device), htk=htk, dtype=dtype, device=device) - - fdiff = mel_f.diff() - ramps = mel_f.unsqueeze(1) - fftfreqs.unsqueeze(0) - - lower = -ramps[:-2] / fdiff[:-1].unsqueeze(1) - weights = lower.minimum(ramps[2:] / fdiff[1:].unsqueeze(1)).clamp(min=0) - - if isinstance(norm, str): - if norm == "slaney": weights *= (2.0 / (mel_f[2:n_mels + 2] - mel_f[:n_mels])).unsqueeze(1) - else: raise ValueError - else: weights = normalize(weights, norm=norm, axis=-1) - - return weights - -def hz_to_mel(frequencies, htk=False, dtype=torch.float32, device=None): - frequencies = torch.as_tensor(frequencies, dtype=dtype, device=device) - if htk: return 2595.0 * (1.0 + frequencies / 700.0).log10() - - f_min = 0.0 - f_sp = 200.0 / 3 - - mels = (frequencies - f_min) / f_sp - - min_log_hz = 1000.0 - min_log_mel = (min_log_hz - f_min) / f_sp - - logstep = torch.tensor(6.4, dtype=dtype, device=device).log() / 27.0 - return torch.where(frequencies >= min_log_hz, min_log_mel + (frequencies / min_log_hz).log() / logstep, mels) - -def mel_to_hz(mels, htk=False, dtype=torch.float32, device=None): - mels = torch.as_tensor(mels, dtype=dtype, device=device) - if htk: return 700.0 * (10.0 ** (mels / 2595.0) - 1.0) - - f_min = 0.0 - f_sp = 200.0 / 3 - - freqs = f_min + f_sp * mels - - min_log_hz = 1000.0 - min_log_mel = (min_log_hz - f_min) / f_sp - logstep = torch.tensor(6.4, dtype=dtype, device=device).log() / 27.0 - - return torch.where(mels >= min_log_mel, min_log_hz * (logstep * (mels - min_log_mel)).exp(), freqs) - -def normalize(S, norm=torch.inf, axis=0, threshold=None, fill=None): - S = torch.as_tensor(S) - - if threshold is None: - x = torch.as_tensor(S) - threshold = torch.finfo(x.dtype if x.dtype in (torch.float16, torch.float32, torch.float64, torch.bfloat16) else torch.float32).tiny - elif threshold <= 0: raise ValueError - - if fill not in [None, False, True] or not torch.all(torch.isfinite(S)): raise ValueError - - mag = S.abs().to(torch.float32) - fill_norm = 1 - - if norm is None: return S - elif norm == torch.inf: length = mag.amax(dim=axis, keepdim=True) - elif norm == -torch.inf: length = mag.amin(dim=axis, keepdim=True) - elif norm == 0: - if fill is True: raise ValueError - length = (mag > 0).sum(dim=axis, keepdim=True, dtype=mag.dtype) - elif isinstance(norm, (int, float)) and norm > 0: - length = (mag ** norm).sum(dim=axis, keepdim=True) ** (1.0 / norm) - fill_norm = (mag.numel() ** (-1.0 / norm)) if axis is None else (mag.shape[axis] ** (-1.0 / norm)) - else: raise ValueError - - small_idx = length < threshold - - if fill is None: - length = length.clone() - length[small_idx] = 1.0 - Snorm = S / length - elif fill: - length = length.clone() - length[small_idx] = torch.nan - Snorm = torch.nan_to_num(S / length, nan=fill_norm) - else: - length = length.clone() - length[small_idx] = torch.inf - Snorm = S / length - - return Snorm \ No newline at end of file diff --git a/main/library/audio/ffmpeg.py b/main/library/audio/ffmpeg.py deleted file mode 100644 index a7b8a4d6b0d69812260b6d8577717a0caeb89cfb..0000000000000000000000000000000000000000 --- a/main/library/audio/ffmpeg.py +++ /dev/null @@ -1,830 +0,0 @@ -import os -import copy -import hashlib -import subprocess - -from collections import namedtuple -from collections.abc import Iterable - -DagEdge = namedtuple( - 'DagEdge', - [ - 'downstream_node', - 'downstream_label', - 'upstream_node', - 'upstream_label', - 'upstream_selector', - ], -) - -class DagNode: - def __hash__(self): - raise NotImplementedError - - def __eq__(self, other): - raise NotImplementedError - - def __repr__(self): - raise NotImplementedError - - @property - def short_repr(self): - raise NotImplementedError - - @property - def incoming_edge_map(self): - raise NotImplementedError - -class KwargReprNode(DagNode): - def __init__(self, incoming_edge_map, name, args=None, kwargs=None): - self._incoming_edge_map = incoming_edge_map - self.name = name - self.args = args or [] - self.kwargs = kwargs or {} - - self._hash = self._build_hash() - def _get_upstream_hashes(self): - hashes = [] - - for downstream_label, upstream_info in self.incoming_edge_map.items(): - upstream_node, upstream_label, upstream_selector = upstream_info - - hashes.extend([ - hash(downstream_label), - hash(upstream_node), - hash(upstream_label), - hash(upstream_selector), - ]) - - return hashes - - def _get_inner_hash(self): - return get_hash({ - 'args': self.args, - 'kwargs': self.kwargs, - }) - - def _build_hash(self): - hashes = self._get_upstream_hashes() - hashes.append(self._get_inner_hash()) - - return get_hash_int(hashes) - - def __hash__(self): - return self._hash - - def __eq__(self, other): - return isinstance(other, self.__class__) and hash(self) == hash(other) - - def __repr__(self): - return self.long_repr() - - @property - def incoming_edge_map(self): - return self._incoming_edge_map - - @property - def incoming_edges(self): - return get_incoming_edges(self, self.incoming_edge_map) - - @property - def short_hash(self): - return format(abs(hash(self)), 'x')[:12] - - @property - def short_repr(self): - return self.name - - def long_repr(self, include_hash=True): - formatted = [repr(arg) for arg in self.args] - - formatted.extend( - '{}={!r}'.format(key, self.kwargs[key]) - for key in sorted(self.kwargs) - ) - - result = '{}({})'.format( - self.name, - ', '.join(formatted), - ) - - if include_hash: - result += ' <{}>'.format(self.short_hash) - - return result - -class Stream: - def __init__( - self, - upstream_node, - upstream_label, - node_types, - upstream_selector=None, - ): - if not is_of_types(upstream_node, node_types): - raise TypeError - - self.node = upstream_node - self.label = upstream_label - self.selector = upstream_selector - - def __hash__(self): - return get_hash_int([ - hash(self.node), - hash(self.label), - ]) - - def __eq__(self, other): - return isinstance(other, Stream) and hash(self) == hash(other) - - def __repr__(self): - node_repr = self.node.long_repr(include_hash=False) - - selector = '' - if self.selector: - selector = ':{}'.format(self.selector) - - return '{}[{!r}{}] <{}>'.format( - node_repr, - self.label, - selector, - self.node.short_hash, - ) - - def __getitem__(self, index): - if self.selector is not None: - raise ValueError - - if not isinstance(index, str): - raise TypeError - - return self.node.stream( - label=self.label, - selector=index, - ) - - @property - def audio(self): - return self['a'] - - @property - def video(self): - return self['v'] - -class Node(KwargReprNode): - @classmethod - def check_input_len(cls, stream_map, min_inputs, max_inputs): - stream_count = len(stream_map) - - if min_inputs is not None and stream_count < min_inputs: - raise ValueError - - if max_inputs is not None and stream_count > max_inputs: - raise ValueError - - @classmethod - def check_input_types(cls, stream_map, incoming_stream_types): - for stream in stream_map.values(): - if not is_of_types(stream, incoming_stream_types): - raise TypeError - - @classmethod - def get_incoming_edge_map(cls, stream_map): - edge_map = {} - - for downstream_label, upstream in stream_map.items(): - edge_map[downstream_label] = ( - upstream.node, - upstream.label, - upstream.selector, - ) - - return edge_map - - def __init__( - self, - stream_spec, - name, - incoming_stream_types, - outgoing_stream_type, - min_inputs, - max_inputs, - args=None, - kwargs=None, - ): - self._outgoing_stream_type = outgoing_stream_type - - stream_map = get_stream_map(stream_spec) - - self.check_input_len( - stream_map, - min_inputs, - max_inputs, - ) - - self.check_input_types( - stream_map, - incoming_stream_types, - ) - - super().__init__( - self.get_incoming_edge_map(stream_map), - name, - args=args, - kwargs=kwargs, - ) - - def stream(self, label=None, selector=None): - return self._outgoing_stream_type( - self, - label, - upstream_selector=selector, - ) - - def __getitem__(self, item): - if isinstance(item, slice): - return self.stream( - label=item.start, - selector=item.stop, - ) - - return self.stream(label=item) - -class FilterableStream(Stream): - def __init__( - self, - upstream_node, - upstream_label, - upstream_selector=None, - ): - super().__init__( - upstream_node, - upstream_label, - {InputNode, FilterNode}, - upstream_selector, - ) - -class InputNode(Node): - def __init__(self, name, args=None, kwargs=None): - super().__init__( - stream_spec=None, - name=name, - incoming_stream_types={}, - outgoing_stream_type=FilterableStream, - min_inputs=0, - max_inputs=0, - args=args, - kwargs=kwargs, - ) - - @property - def short_repr(self): - return os.path.basename(self.kwargs['filename']) - -class FilterNode(Node): - def __init__( - self, - stream_spec, - name, - max_inputs=1, - args=None, - kwargs=None, - ): - super().__init__( - stream_spec=stream_spec, - name=name, - incoming_stream_types={FilterableStream}, - outgoing_stream_type=FilterableStream, - min_inputs=1, - max_inputs=max_inputs, - args=args, - kwargs=kwargs, - ) - - def _get_filter(self, outgoing_edges): - args = self.args - kwargs = self.kwargs - - if self.name in ('split', 'asplit'): - args = [len(outgoing_edges)] - - escaped_args = [ - escape_chars(value, "\\'=:") - for value in args - ] - - escaped_kwargs = {} - - for key, value in kwargs.items(): - escaped_kwargs[ - escape_chars(key, "\\'=:") - ] = escape_chars(value, "\\'=:") - - params = [] - - params.extend( - escape_chars(value, "\\'=:") - for value in escaped_args - ) - - params.extend( - '{}={}'.format(key, escaped_kwargs[key]) - for key in sorted(escaped_kwargs) - ) - - result = escape_chars(self.name, "\\'=:") - if params: result += '={}'.format(':'.join(params)) - - return escape_chars(result, "\\'[],;") - -class OutputNode(Node): - def __init__(self, stream, name, args=None, kwargs=None): - super().__init__( - stream_spec=stream, - name=name, - incoming_stream_types={FilterableStream}, - outgoing_stream_type=OutputStream, - min_inputs=1, - max_inputs=None, - args=args, - kwargs=kwargs, - ) - - @property - def short_repr(self): - return os.path.basename(self.kwargs['filename']) - -class OutputStream(Stream): - def __init__( - self, - upstream_node, - upstream_label, - upstream_selector=None, - ): - super().__init__( - upstream_node, - upstream_label, - {OutputNode, GlobalNode, MergeOutputsNode}, - upstream_selector, - ) - -class MergeOutputsNode(Node): - def __init__(self, streams, name): - super().__init__( - stream_spec=streams, - name=name, - incoming_stream_types={OutputStream}, - outgoing_stream_type=OutputStream, - min_inputs=1, - max_inputs=None, - ) - -class GlobalNode(Node): - def __init__(self, stream, name, args=None, kwargs=None): - super().__init__( - stream_spec=stream, - name=name, - incoming_stream_types={OutputStream}, - outgoing_stream_type=OutputStream, - min_inputs=1, - max_inputs=1, - args=args, - kwargs=kwargs, - ) - -def get_incoming_edges(downstream_node, incoming_edge_map): - edges = [] - - for downstream_label, upstream_info in incoming_edge_map.items(): - upstream_node, upstream_label, upstream_selector = upstream_info - - edges.append( - DagEdge( - downstream_node, - downstream_label, - upstream_node, - upstream_label, - upstream_selector, - ) - ) - - return edges - -def get_outgoing_edges(upstream_node, outgoing_edge_map): - edges = [] - - for upstream_label, downstream_infos in sorted(outgoing_edge_map.items()): - for downstream_info in downstream_infos: - downstream_node, downstream_label, downstream_selector = downstream_info - - edges.append( - DagEdge( - downstream_node, - downstream_label, - upstream_node, - upstream_label, - downstream_selector, - ) - ) - - return edges - -def is_of_types(obj, types): - return any(isinstance(obj, item) for item in types) - -def get_stream_map(stream_spec): - if stream_spec is None: - return {} - - if isinstance(stream_spec, Stream): - return {None: stream_spec} - - if isinstance(stream_spec, (list, tuple)): - return dict(enumerate(stream_spec)) - - if isinstance(stream_spec, dict): - return stream_spec - - raise TypeError - -def get_stream_map_nodes(stream_map): - nodes = [] - - for stream in stream_map.values(): - if not isinstance(stream, Stream): - raise TypeError - - nodes.append(stream.node) - - return nodes - -def get_stream_spec_nodes(stream_spec): - return get_stream_map_nodes( - get_stream_map(stream_spec) - ) - -def stream_operator(stream_classes=None, name=None): - stream_classes = stream_classes or {Stream} - - def decorator(func): - func_name = name or func.__name__ - - for stream_class in stream_classes: - setattr(stream_class, func_name, func) - - return func - - return decorator - -def filter_operator(name=None): - return stream_operator( - stream_classes={FilterableStream}, - name=name, - ) - -def output_operator(name=None): - return stream_operator( - stream_classes={OutputStream}, - name=name, - ) - -def input(filename, **kwargs): - kwargs['filename'] = filename - - fmt = kwargs.pop('f', None) - - if fmt: - if 'format' in kwargs: - raise ValueError - - kwargs['format'] = fmt - - return InputNode( - input.__name__, - kwargs=kwargs, - ).stream() - -@output_operator() -def global_args(stream, *args): - return GlobalNode( - stream, - global_args.__name__, - args, - ).stream() - -@output_operator() -def overwrite_output(stream): - return GlobalNode( - stream, - overwrite_output.__name__, - ['-y'], - ).stream() - -@output_operator() -def merge_outputs(*streams): - return MergeOutputsNode( - streams, - merge_outputs.__name__, - ).stream() - -@filter_operator() -def output(*streams_and_filename, **kwargs): - streams_and_filename = list(streams_and_filename) - - if 'filename' not in kwargs: - if not isinstance(streams_and_filename[-1], str): - raise ValueError - - kwargs['filename'] = streams_and_filename.pop() - - fmt = kwargs.pop('f', None) - - if fmt: - if 'format' in kwargs: - raise ValueError - - kwargs['format'] = fmt - - return OutputNode( - streams_and_filename, - output.__name__, - kwargs=kwargs, - ).stream() - -def escape_chars(text, chars): - text = str(text) - - chars = list(set(chars)) - - if '\\' in chars: - chars.remove('\\') - chars.insert(0, '\\') - - for char in chars: - text = text.replace(char, '\\' + char) - - return text - -def recursive_repr(item): - if isinstance(item, str): - return item - - if isinstance(item, list): - return '[{}]'.format( - ', '.join(recursive_repr(x) for x in item) - ) - - if isinstance(item, dict): - return '{{{}}}'.format( - ', '.join( - '{}: {}'.format( - recursive_repr(key), - recursive_repr(item[key]), - ) - for key in sorted(item) - ) - ) - - return repr(item) - -def get_hash(item): - value = recursive_repr(item).encode('utf-8') - - return hashlib.md5(value).hexdigest() - -def get_hash_int(item): - return int(get_hash(item), 16) - -@output_operator() -def run( - stream_spec, - cmd='ffmpeg', - capture_stdout=False, - capture_stderr=False, - input=None, - quiet=False, - overwrite_output=False, - cwd=None, -): - process = run_async( - stream_spec, - cmd=cmd, - pipe_stdin=input is not None, - pipe_stdout=capture_stdout, - pipe_stderr=capture_stderr, - quiet=quiet, - overwrite_output=overwrite_output, - cwd=cwd, - ) - - out, err = process.communicate(input) - retcode = process.poll() - - if retcode: raise RuntimeError(err) - return out, err - -@output_operator() -def run_async( - stream_spec, - cmd='ffmpeg', - pipe_stdin=False, - pipe_stdout=False, - pipe_stderr=False, - quiet=False, - overwrite_output=False, - cwd=None -): - args = compile(stream_spec, cmd=cmd, overwrite_output=overwrite_output) - - stdin_stream = subprocess.PIPE if pipe_stdin else None - stdout_stream = subprocess.PIPE if pipe_stdout else None - stderr_stream = subprocess.PIPE if pipe_stderr else None - - if quiet: - stdout_stream = subprocess.DEVNULL - stderr_stream = subprocess.STDOUT - - return subprocess.Popen(args, stdin=stdin_stream, stdout=stdout_stream, stderr=stderr_stream, cwd=cwd) - -@output_operator() -def compile(stream_spec, cmd='ffmpeg', overwrite_output=False): - if isinstance(cmd, str): cmd = [cmd] - elif not isinstance(cmd, list): cmd = list(cmd) - - return cmd + get_args(stream_spec, overwrite_output=overwrite_output) - -@output_operator() -def get_args(stream_spec, overwrite_output=False): - nodes = get_stream_spec_nodes(stream_spec) - sorted_nodes, outgoing_edge_maps = topo_sort(nodes) - args, input_nodes, output_nodes, global_nodes, filter_nodes = [], [], [], [], [] - - for node in sorted_nodes: - if isinstance(node, InputNode): input_nodes.append(node) - elif isinstance(node, OutputNode): output_nodes.append(node) - elif isinstance(node, GlobalNode): global_nodes.append(node) - elif isinstance(node, FilterNode): filter_nodes.append(node) - - stream_name_map = {(node, None): str(index) for index, node in enumerate(input_nodes)} - for node in input_nodes: - args.extend(get_input_args(node)) - - filter_arg = get_filter_arg(filter_nodes, outgoing_edge_maps, stream_name_map) - - if filter_arg: args.extend(['-filter_complex', filter_arg]) - - for node in output_nodes: - args.extend(get_output_args(node, stream_name_map)) - - for node in global_nodes: - args.extend(get_global_args(node)) - - if overwrite_output: args.append('-y') - return args - -def get_global_args(node): - return list(node.args) - -def get_filter_arg(filter_nodes, outgoing_edge_maps, stream_name_map): - allocate_filter_stream_names(filter_nodes, outgoing_edge_maps, stream_name_map) - return ';'.join([get_filter_spec(node, outgoing_edge_maps[node], stream_name_map) for node in filter_nodes]) - -def allocate_filter_stream_names(filter_nodes, outgoing_edge_maps, stream_name_map): - stream_count = 0 - - for upstream_node in filter_nodes: - outgoing_edge_map = outgoing_edge_maps[upstream_node] - - for upstream_label, downstreams in sorted(outgoing_edge_map.items()): - if len(downstreams) > 1: raise ValueError - - stream_name_map[(upstream_node, upstream_label)] = 's{}'.format(stream_count) - stream_count += 1 - -def get_filter_spec(node, outgoing_edge_map, stream_name_map): - incoming_edges = node.incoming_edges - outgoing_edges = get_outgoing_edges(node, outgoing_edge_map) - - inputs = [format_input_stream_name(stream_name_map, edge) for edge in incoming_edges] - outputs = [format_output_stream_name(stream_name_map, edge) for edge in outgoing_edges] - - return '{}{}{}'.format(''.join(inputs), node._get_filter(outgoing_edges), ''.join(outputs)) - -def get_input_args(input_node): - if input_node.name != input.__name__: raise ValueError - - kwargs = copy.copy(input_node.kwargs) - filename = kwargs.pop('filename') - fmt = kwargs.pop('format', None) - video_size = kwargs.pop('video_size', None) - args = [] - - if fmt: args.extend(['-f', fmt]) - if video_size: args.extend(['-video_size', '{}x{}'.format(video_size[0], video_size[1])]) - - args.extend(convert_kwargs_to_cmd_line_args(kwargs)) - args.extend(['-i', filename]) - - return args - -def get_output_args(node, stream_name_map): - if node.name != output.__name__: raise ValueError - if not node.incoming_edges: raise ValueError - - args = [] - - for edge in node.incoming_edges: - stream_name = format_input_stream_name(stream_name_map, edge, is_final_arg=True) - if stream_name != '0' or len(node.incoming_edges) > 1: args.extend(['-map', stream_name]) - - kwargs = copy.copy(node.kwargs) - filename = kwargs.pop('filename') - - if 'format' in kwargs: args.extend(['-f', kwargs.pop('format')]) - if 'video_bitrate' in kwargs: args.extend(['-b:v', str(kwargs.pop('video_bitrate'))]) - if 'audio_bitrate' in kwargs: args.extend(['-b:a', str(kwargs.pop('audio_bitrate'))]) - - if 'video_size' in kwargs: - video_size = kwargs.pop('video_size') - if not isinstance(video_size, str) and isinstance(video_size, Iterable): video_size = '{}x{}'.format(video_size[0], video_size[1]) - - args.extend(['-video_size', video_size]) - - args.extend(convert_kwargs_to_cmd_line_args(kwargs)) - args.append(filename) - - return args - -def convert_kwargs_to_cmd_line_args(kwargs): - args = [] - - for key in sorted(kwargs): - value = kwargs[key] - - if isinstance(value, Iterable) and not isinstance(value, str): - for item in value: - args.append('-{}'.format(key)) - if item is not None: args.append(str(item)) - - continue - - args.append('-{}'.format(key)) - if value is not None: args.append(str(value)) - - return args - -def format_input_stream_name(stream_name_map, edge, is_final_arg=False): - prefix = stream_name_map[(edge.upstream_node, edge.upstream_label)] - suffix = '' - - if edge.upstream_selector: suffix = ':{}'.format(edge.upstream_selector) - if is_final_arg and isinstance(edge.upstream_node, InputNode): return '{}{}'.format(prefix, suffix) - - return '[{}{}]'.format(prefix, suffix) - -def format_output_stream_name(stream_name_map, edge): - return '[{}]'.format(stream_name_map[(edge.upstream_node, edge.upstream_label)]) - -def topo_sort(downstream_nodes): - marked_nodes = set() - sorted_nodes = [] - outgoing_edge_maps = {} - - def visit( - upstream_node, - upstream_label, - downstream_node, - downstream_label, - downstream_selector=None - ): - if upstream_node in marked_nodes: raise RuntimeError - - if downstream_node is not None: - outgoing_edge_map = outgoing_edge_maps.setdefault( - upstream_node, - {} - ) - - outgoing_edges = outgoing_edge_map.setdefault( - upstream_label, - [] - ) - - outgoing_edges.append((downstream_node, downstream_label, downstream_selector)) - - if upstream_node in sorted_nodes: return - marked_nodes.add(upstream_node) - - for edge in upstream_node.incoming_edges: - visit(edge.upstream_node, edge.upstream_label, edge.downstream_node, edge.downstream_label, edge.upstream_selector) - - marked_nodes.remove(upstream_node) - sorted_nodes.append(upstream_node) - - pending_nodes = [(node, None) for node in downstream_nodes] - - while pending_nodes: - upstream_node, upstream_label = pending_nodes.pop() - visit(upstream_node, upstream_label, None, None) - - return sorted_nodes, outgoing_edge_maps \ No newline at end of file diff --git a/main/library/audio/noisereduce.py b/main/library/audio/noisereduce.py deleted file mode 100644 index 0b93b3dc04549bb5f981393710b843a7e1bbde9f..0000000000000000000000000000000000000000 --- a/main/library/audio/noisereduce.py +++ /dev/null @@ -1,198 +0,0 @@ -import os -import sys -import torch - -from torch.nn.functional import conv1d, conv2d - -sys.path.append(os.getcwd()) - -from main.app.variables import config - -@torch.no_grad() -def temperature_sigmoid(x, x0, temp_coeff): - return ((x - x0) / temp_coeff).sigmoid() - -@torch.no_grad() -def linspace(start, stop, num = 50, endpoint = True, **kwargs): - return ( - torch.linspace( - start, - stop, - num, - **kwargs - ) - ) if endpoint else ( - torch.linspace( - start, - stop, - num + 1, - **kwargs - )[:-1] - ) - -@torch.no_grad() -def amp_to_db(x, eps=torch.finfo(torch.float32).eps, top_db=40): - x_db = 20 * (x + eps).log10() - - return x_db.max( - (x_db.max(-1).values - top_db).unsqueeze(-1) - ) - -class TorchGate(torch.nn.Module): - @torch.no_grad() - def __init__( - self, - sr, - nonstationary = False, - n_std_thresh_stationary = 1.5, - n_thresh_nonstationary = 1.3, - temp_coeff_nonstationary = 0.1, - n_movemean_nonstationary = 20, - prop_decrease = 1.0, - n_fft = 1024, - win_length = None, - hop_length = None, - freq_mask_smooth_hz = 500, - time_mask_smooth_ms = 50 - ): - super().__init__() - self.sr = sr - self.nonstationary = nonstationary - assert 0.0 <= prop_decrease <= 1.0 - self.prop_decrease = prop_decrease - self.n_fft = n_fft - self.win_length = self.n_fft if win_length is None else win_length - self.hop_length = self.win_length // 4 if hop_length is None else hop_length - self.n_std_thresh_stationary = n_std_thresh_stationary - self.temp_coeff_nonstationary = temp_coeff_nonstationary - self.n_movemean_nonstationary = n_movemean_nonstationary - self.n_thresh_nonstationary = n_thresh_nonstationary - self.freq_mask_smooth_hz = freq_mask_smooth_hz - self.time_mask_smooth_ms = time_mask_smooth_ms - self.register_buffer("smoothing_filter", self._generate_mask_smoothing_filter()) - self.forward = self._forward_other_backends if config.device.startswith(("ocl", "privateuseone")) else self._forward_torch - - @torch.no_grad() - def _generate_mask_smoothing_filter(self): - if self.freq_mask_smooth_hz is None and self.time_mask_smooth_ms is None: return None - n_grad_freq = (1 if self.freq_mask_smooth_hz is None else int(self.freq_mask_smooth_hz / (self.sr / (self.n_fft / 2)))) - if n_grad_freq < 1: raise ValueError - - n_grad_time = (1 if self.time_mask_smooth_ms is None else int(self.time_mask_smooth_ms / ((self.hop_length / self.sr) * 1000))) - if n_grad_time < 1: raise ValueError - if n_grad_time == 1 and n_grad_freq == 1: return None - - smoothing_filter = torch.outer( - torch.cat([ - linspace(0, 1, n_grad_freq + 1, endpoint=False), - linspace(1, 0, n_grad_freq + 2) - ])[1:-1], - torch.cat([ - linspace(0, 1, n_grad_time + 1, endpoint=False), - linspace(1, 0, n_grad_time + 2) - ])[1:-1] - ).unsqueeze(0).unsqueeze(0) - - return smoothing_filter / smoothing_filter.sum() - - @torch.no_grad() - def _stationary_mask(self, X_db): - std_freq_noise, mean_freq_noise = torch.std_mean(X_db, dim=-1) - return X_db > (mean_freq_noise + std_freq_noise * self.n_std_thresh_stationary).unsqueeze(2) - - @torch.no_grad() - def _nonstationary_mask(self, X_abs): - X_smoothed = ( - conv1d( - X_abs.reshape(-1, 1, X_abs.shape[-1]), - torch.ones( - self.n_movemean_nonstationary, - dtype=X_abs.dtype, - device=X_abs.device - ).view(1, 1, -1), - padding="same" - ).view(X_abs.shape) / self.n_movemean_nonstationary - ) - - return temperature_sigmoid( - ((X_abs - X_smoothed) / X_smoothed), - self.n_thresh_nonstationary, - self.temp_coeff_nonstationary - ) - - def _forward_torch(self, x): - assert x.ndim == 2 - if x.shape[-1] < self.win_length * 2: raise Exception - - X = torch.stft( - x, - n_fft=self.n_fft, - hop_length=self.hop_length, - win_length=self.win_length, - return_complex=True, - pad_mode="constant", - center=True, - window=torch.hann_window(self.win_length).to(x.device) - ) - - sig_mask = self._nonstationary_mask(X.abs()) if self.nonstationary else self._stationary_mask(amp_to_db(X.abs())) - sig_mask = self.prop_decrease * (sig_mask.float() * 1.0 - 1.0) + 1.0 - - if self.smoothing_filter is not None: - sig_mask = conv2d( - sig_mask.unsqueeze(1), - self.smoothing_filter.to(sig_mask.dtype), - padding="same" - ) - - Y = X * sig_mask.squeeze(1) - - return ( - torch.istft( - Y, - n_fft=self.n_fft, - hop_length=self.hop_length, - win_length=self.win_length, - center=True, - window=torch.hann_window(self.win_length).to(Y.device) - ).to(dtype=x.dtype) - ) - - def _forward_other_backends(self, x): - assert x.ndim == 2 - if x.shape[-1] < self.win_length * 2: raise Exception - - if not hasattr(self, "stft"): - from main.library.backends.utils import STFT - - self.stft = STFT( - filter_length=self.n_fft, - hop_length=self.hop_length, - win_length=self.win_length, - pad_mode="constant" - ).to(x.device) - - X, phase = self.stft.transform( - x, - eps=1e-9, - return_phase=True - ) - - sig_mask = self._nonstationary_mask(X.abs()) if self.nonstationary else self._stationary_mask(amp_to_db(X.abs())) - sig_mask = self.prop_decrease * (sig_mask.float() * 1.0 - 1.0) + 1.0 - - if self.smoothing_filter is not None: - sig_mask = conv2d( - sig_mask.unsqueeze(1), - self.smoothing_filter.to(sig_mask.dtype), - padding="same" - ) - - Y = X * sig_mask.squeeze(1) - - return ( - self.stft.inverse( - Y, - phase - ) - ) \ No newline at end of file diff --git a/main/library/audio/sounddevice.py b/main/library/audio/sounddevice.py deleted file mode 100644 index f8bd5ef1b3c66c7c6f526e499bc49fa8fb4767db..0000000000000000000000000000000000000000 --- a/main/library/audio/sounddevice.py +++ /dev/null @@ -1,1011 +0,0 @@ -import os -import sys -import pickle -import locale -import atexit -import platform -import threading -import contextlib -import _cffi_backend - -import numpy as np - -from numbers import Integral -from ctypes.util import find_library - -sys.path.append(os.getcwd()) - -from main.app.variables import configs, translations - -ffi = _cffi_backend.FFI('portaudio', - _version = 0x2601, - _types = b'\x00\x00\x76\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x79\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x1C\x0D\x00\x00\x8D\x03\x00\x00\x00\x0F\x00\x00\x7B\x0D\x00\x00\x00\x0F\x00\x00\x80\x0D\x00\x00\x07\x11\x00\x00\x00\x0F\x00\x00\x88\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x88\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x01\x01\x00\x00\x00\x0F\x00\x00\x88\x0D\x00\x00\x00\x0F\x00\x00\x21\x0D\x00\x00\x07\x11\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x0B\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x82\x03\x00\x00\x1F\x11\x00\x00\x0E\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x03\x00\x00\x1F\x11\x00\x00\x1F\x11\x00\x00\x0E\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x52\x03\x00\x00\x07\x11\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x2E\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x0E\x01\x00\x00\x0A\x01\x00\x00\x34\x11\x00\x00\x07\x11\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x11\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x11\x00\x00\x07\x11\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x11\x00\x00\x8D\x03\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x11\x00\x00\x6B\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x4B\x11\x00\x00\x07\x11\x00\x00\x0A\x01\x00\x00\x7F\x03\x00\x00\x0A\x01\x00\x00\x07\x11\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x69\x0D\x00\x00\x07\x11\x00\x00\x00\x0F\x00\x00\x8D\x0D\x00\x00\x7D\x03\x00\x00\x8B\x03\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x8D\x0D\x00\x00\x60\x11\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x8D\x0D\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x8D\x0D\x00\x00\x07\x11\x00\x00\x00\x0F\x00\x00\x8D\x0D\x00\x00\x07\x11\x00\x00\x09\x01\x00\x00\x07\x11\x00\x00\x09\x01\x00\x00\x07\x11\x00\x00\x00\x0F\x00\x00\x01\x09\x00\x00\x77\x03\x00\x00\x02\x09\x00\x00\x00\x0B\x00\x00\x7A\x03\x00\x00\x03\x09\x00\x00\x7C\x03\x00\x00\x04\x09\x00\x00\x00\x09\x00\x00\x02\x0B\x00\x00\x05\x09\x00\x00\x81\x03\x00\x00\x06\x09\x00\x00\x07\x09\x00\x00\x03\x0B\x00\x00\x04\x0B\x00\x00\x08\x09\x00\x00\x05\x0B\x00\x00\x06\x0B\x00\x00\x89\x03\x00\x00\x02\x01\x00\x00\x01\x03\x00\x00\x15\x01\x00\x00\x6E\x03\x00\x00\x00\x01', - _globals = (b'\x00\x00\x11\x23PaMacCore_GetChannelName',0,b'\x00\x00\x5F\x23PaMacCore_SetupChannelMap',0,b'\x00\x00\x64\x23PaMacCore_SetupStreamInfo',0,b'\x00\x00\x23\x23PaWasapi_IsLoopback',0,b'\x00\x00\x5A\x23PaWasapi_UpdateDeviceList',0,b'\x00\x00\x41\x23Pa_AbortStream',0,b'\x00\x00\x41\x23Pa_CloseStream',0,b'\x00\x00\x5A\x23Pa_GetDefaultHostApi',0,b'\x00\x00\x5A\x23Pa_GetDefaultInputDevice',0,b'\x00\x00\x5A\x23Pa_GetDefaultOutputDevice',0,b'\x00\x00\x5A\x23Pa_GetDeviceCount',0,b'\x00\x00\x00\x23Pa_GetDeviceInfo',0,b'\x00\x00\x0E\x23Pa_GetErrorText',0,b'\x00\x00\x5A\x23Pa_GetHostApiCount',0,b'\x00\x00\x03\x23Pa_GetHostApiInfo',0,b'\x00\x00\x09\x23Pa_GetLastHostErrorInfo',0,b'\x00\x00\x2A\x23Pa_GetSampleSize',0,b'\x00\x00\x18\x23Pa_GetStreamCpuLoad',0,b'\x00\x00\x06\x23Pa_GetStreamHostApiType',0,b'\x00\x00\x0B\x23Pa_GetStreamInfo',0,b'\x00\x00\x5C\x23Pa_GetStreamReadAvailable',0,b'\x00\x00\x18\x23Pa_GetStreamTime',0,b'\x00\x00\x5C\x23Pa_GetStreamWriteAvailable',0,b'\x00\x00\x5A\x23Pa_GetVersion',0,b'\x00\x00\x16\x23Pa_GetVersionText',0,b'\x00\x00\x26\x23Pa_HostApiDeviceIndexToDeviceIndex',0,b'\x00\x00\x1B\x23Pa_HostApiTypeIdToHostApiIndex',0,b'\x00\x00\x5A\x23Pa_Initialize',0,b'\x00\x00\x1E\x23Pa_IsFormatSupported',0,b'\x00\x00\x41\x23Pa_IsStreamActive',0,b'\x00\x00\x41\x23Pa_IsStreamStopped',0,b'\x00\x00\x37\x23Pa_OpenDefaultStream',0,b'\x00\x00\x2D\x23Pa_OpenStream',0,b'\x00\x00\x44\x23Pa_ReadStream',0,b'\x00\x00\x4E\x23Pa_SetStreamFinishedCallback',0,b'\x00\x00\x68\x23Pa_Sleep',0,b'\x00\x00\x41\x23Pa_StartStream',0,b'\x00\x00\x41\x23Pa_StopStream',0,b'\x00\x00\x5A\x23Pa_Terminate',0,b'\x00\x00\x49\x23Pa_WriteStream',0,b'\xFF\xFF\xFF\x0BeAudioCategoryAlerts',4,b'\xFF\xFF\xFF\x0BeAudioCategoryCommunications',3,b'\xFF\xFF\xFF\x0BeAudioCategoryGameChat',8,b'\xFF\xFF\xFF\x0BeAudioCategoryGameEffects',6,b'\xFF\xFF\xFF\x0BeAudioCategoryGameMedia',7,b'\xFF\xFF\xFF\x0BeAudioCategoryMedia',11,b'\xFF\xFF\xFF\x0BeAudioCategoryMovie',10,b'\xFF\xFF\xFF\x0BeAudioCategoryOther',0,b'\xFF\xFF\xFF\x0BeAudioCategorySoundEffects',5,b'\xFF\xFF\xFF\x0BeAudioCategorySpeech',9,b'\xFF\xFF\xFF\x0BeStreamOptionMatchFormat',2,b'\xFF\xFF\xFF\x0BeStreamOptionNone',0,b'\xFF\xFF\xFF\x0BeStreamOptionRaw',1,b'\xFF\xFF\xFF\x0BeThreadPriorityAudio',1,b'\xFF\xFF\xFF\x0BeThreadPriorityCapture',2,b'\xFF\xFF\xFF\x0BeThreadPriorityDistribution',3,b'\xFF\xFF\xFF\x0BeThreadPriorityGames',4,b'\xFF\xFF\xFF\x0BeThreadPriorityNone',0,b'\xFF\xFF\xFF\x0BeThreadPriorityPlayback',5,b'\xFF\xFF\xFF\x0BeThreadPriorityProAudio',6,b'\xFF\xFF\xFF\x0BeThreadPriorityWindowManager',7,b'\xFF\xFF\xFF\x0BpaAL',9,b'\xFF\xFF\xFF\x0BpaALSA',8,b'\xFF\xFF\xFF\x0BpaASIO',3,b'\xFF\xFF\xFF\x0BpaAbort',2,b'\xFF\xFF\xFF\x1FpaAsioUseChannelSelectors',1,b'\xFF\xFF\xFF\x0BpaAudioScienceHPI',14,b'\xFF\xFF\xFF\x0BpaBadBufferPtr',-9972,b'\xFF\xFF\xFF\x0BpaBadIODeviceCombination',-9993,b'\xFF\xFF\xFF\x0BpaBadStreamPtr',-9988,b'\xFF\xFF\xFF\x0BpaBeOS',10,b'\xFF\xFF\xFF\x0BpaBufferTooBig',-9991,b'\xFF\xFF\xFF\x0BpaBufferTooSmall',-9990,b'\xFF\xFF\xFF\x0BpaCanNotReadFromACallbackStream',-9977,b'\xFF\xFF\xFF\x0BpaCanNotReadFromAnOutputOnlyStream',-9975,b'\xFF\xFF\xFF\x0BpaCanNotWriteToACallbackStream',-9976,b'\xFF\xFF\xFF\x0BpaCanNotWriteToAnInputOnlyStream',-9974,b'\xFF\xFF\xFF\x1FpaClipOff',1,b'\xFF\xFF\xFF\x0BpaComplete',1,b'\xFF\xFF\xFF\x0BpaContinue',0,b'\xFF\xFF\xFF\x0BpaCoreAudio',5,b'\xFF\xFF\xFF\x1FpaCustomFormat',65536,b'\xFF\xFF\xFF\x0BpaDeviceUnavailable',-9985,b'\xFF\xFF\xFF\x0BpaDirectSound',1,b'\xFF\xFF\xFF\x1FpaDitherOff',2,b'\xFF\xFF\xFF\x1FpaFloat32',1,b'\xFF\xFF\xFF\x1FpaFormatIsSupported',0,b'\xFF\xFF\xFF\x1FpaFramesPerBufferUnspecified',0,b'\xFF\xFF\xFF\x0BpaHostApiNotFound',-9979,b'\xFF\xFF\xFF\x0BpaInDevelopment',0,b'\xFF\xFF\xFF\x0BpaIncompatibleHostApiSpecificStreamInfo',-9984,b'\xFF\xFF\xFF\x0BpaIncompatibleStreamHostApi',-9973,b'\xFF\xFF\xFF\x1FpaInputOverflow',2,b'\xFF\xFF\xFF\x0BpaInputOverflowed',-9981,b'\xFF\xFF\xFF\x1FpaInputUnderflow',1,b'\xFF\xFF\xFF\x0BpaInsufficientMemory',-9992,b'\xFF\xFF\xFF\x1FpaInt16',8,b'\xFF\xFF\xFF\x1FpaInt24',4,b'\xFF\xFF\xFF\x1FpaInt32',2,b'\xFF\xFF\xFF\x1FpaInt8',16,b'\xFF\xFF\xFF\x0BpaInternalError',-9986,b'\xFF\xFF\xFF\x0BpaInvalidChannelCount',-9998,b'\xFF\xFF\xFF\x0BpaInvalidDevice',-9996,b'\xFF\xFF\xFF\x0BpaInvalidFlag',-9995,b'\xFF\xFF\xFF\x0BpaInvalidHostApi',-9978,b'\xFF\xFF\xFF\x0BpaInvalidSampleRate',-9997,b'\xFF\xFF\xFF\x0BpaJACK',12,b'\xFF\xFF\xFF\x0BpaMME',2,b'\xFF\xFF\xFF\x1FpaMacCoreChangeDeviceParameters',1,b'\xFF\xFF\xFF\x1FpaMacCoreConversionQualityHigh',1024,b'\xFF\xFF\xFF\x1FpaMacCoreConversionQualityLow',768,b'\xFF\xFF\xFF\x1FpaMacCoreConversionQualityMax',0,b'\xFF\xFF\xFF\x1FpaMacCoreConversionQualityMedium',512,b'\xFF\xFF\xFF\x1FpaMacCoreConversionQualityMin',256,b'\xFF\xFF\xFF\x1FpaMacCoreFailIfConversionRequired',2,b'\xFF\xFF\xFF\x1FpaMacCoreMinimizeCPU',257,b'\xFF\xFF\xFF\x1FpaMacCoreMinimizeCPUButPlayNice',256,b'\xFF\xFF\xFF\x1FpaMacCorePlayNice',0,b'\xFF\xFF\xFF\x1FpaMacCorePro',1,b'\xFF\xFF\xFF\x1FpaNeverDropInput',4,b'\xFF\xFF\xFF\x1FpaNoDevice',-1,b'\xFF\xFF\xFF\x0BpaNoError',0,b'\xFF\xFF\xFF\x1FpaNoFlag',0,b'\xFF\xFF\xFF\x1FpaNonInterleaved',2147483648,b'\xFF\xFF\xFF\x0BpaNotInitialized',-10000,b'\xFF\xFF\xFF\x0BpaNullCallback',-9989,b'\xFF\xFF\xFF\x0BpaOSS',7,b'\xFF\xFF\xFF\x1FpaOutputOverflow',8,b'\xFF\xFF\xFF\x1FpaOutputUnderflow',4,b'\xFF\xFF\xFF\x0BpaOutputUnderflowed',-9980,b'\xFF\xFF\xFF\x1FpaPlatformSpecificFlags',4294901760,b'\xFF\xFF\xFF\x1FpaPrimeOutputBuffersUsingStreamCallback',8,b'\xFF\xFF\xFF\x1FpaPrimingOutput',16,b'\xFF\xFF\xFF\x0BpaSampleFormatNotSupported',-9994,b'\xFF\xFF\xFF\x0BpaSoundManager',4,b'\xFF\xFF\xFF\x0BpaStreamIsNotStopped',-9982,b'\xFF\xFF\xFF\x0BpaStreamIsStopped',-9983,b'\xFF\xFF\xFF\x0BpaTimedOut',-9987,b'\xFF\xFF\xFF\x1FpaUInt8',32,b'\xFF\xFF\xFF\x0BpaUnanticipatedHostError',-9999,b'\xFF\xFF\xFF\x1FpaUseHostApiSpecificDeviceSpecification',-2,b'\xFF\xFF\xFF\x0BpaWASAPI',13,b'\xFF\xFF\xFF\x0BpaWDMKS',11,b'\xFF\xFF\xFF\x0BpaWinWasapiAutoConvert',64,b'\xFF\xFF\xFF\x0BpaWinWasapiExclusive',1,b'\xFF\xFF\xFF\x0BpaWinWasapiExplicitSampleFormat',32,b'\xFF\xFF\xFF\x0BpaWinWasapiPolling',8,b'\xFF\xFF\xFF\x0BpaWinWasapiRedirectHostProcessor',2,b'\xFF\xFF\xFF\x0BpaWinWasapiThreadPriority',16,b'\xFF\xFF\xFF\x0BpaWinWasapiUseChannelMask',4), - _struct_unions = ((b'\x00\x00\x00\x7D\x00\x00\x00\x02$PaMacCoreStreamInfo',b'\x00\x00\x2B\x11size',b'\x00\x00\x1C\x11hostApiType',b'\x00\x00\x2B\x11version',b'\x00\x00\x2B\x11flags',b'\x00\x00\x61\x11channelMap',b'\x00\x00\x2B\x11channelMapSize'),(b'\x00\x00\x00\x75\x00\x00\x00\x02PaAsioStreamInfo',b'\x00\x00\x2B\x11size',b'\x00\x00\x1C\x11hostApiType',b'\x00\x00\x2B\x11version',b'\x00\x00\x2B\x11flags',b'\x00\x00\x8A\x11channelSelectors'),(b'\x00\x00\x00\x77\x00\x00\x00\x02PaDeviceInfo',b'\x00\x00\x01\x11structVersion',b'\x00\x00\x88\x11name',b'\x00\x00\x01\x11hostApi',b'\x00\x00\x01\x11maxInputChannels',b'\x00\x00\x01\x11maxOutputChannels',b'\x00\x00\x21\x11defaultLowInputLatency',b'\x00\x00\x21\x11defaultLowOutputLatency',b'\x00\x00\x21\x11defaultHighInputLatency',b'\x00\x00\x21\x11defaultHighOutputLatency',b'\x00\x00\x21\x11defaultSampleRate'),(b'\x00\x00\x00\x7A\x00\x00\x00\x02PaHostApiInfo',b'\x00\x00\x01\x11structVersion',b'\x00\x00\x1C\x11type',b'\x00\x00\x88\x11name',b'\x00\x00\x01\x11deviceCount',b'\x00\x00\x01\x11defaultInputDevice',b'\x00\x00\x01\x11defaultOutputDevice'),(b'\x00\x00\x00\x7C\x00\x00\x00\x02PaHostErrorInfo',b'\x00\x00\x1C\x11hostApiType',b'\x00\x00\x69\x11errorCode',b'\x00\x00\x88\x11errorText'),(b'\x00\x00\x00\x7F\x00\x00\x00\x02PaStreamCallbackTimeInfo',b'\x00\x00\x21\x11inputBufferAdcTime',b'\x00\x00\x21\x11currentTime',b'\x00\x00\x21\x11outputBufferDacTime'),(b'\x00\x00\x00\x81\x00\x00\x00\x02PaStreamInfo',b'\x00\x00\x01\x11structVersion',b'\x00\x00\x21\x11inputLatency',b'\x00\x00\x21\x11outputLatency',b'\x00\x00\x21\x11sampleRate'),(b'\x00\x00\x00\x82\x00\x00\x00\x02PaStreamParameters',b'\x00\x00\x01\x11device',b'\x00\x00\x01\x11channelCount',b'\x00\x00\x2B\x11sampleFormat',b'\x00\x00\x21\x11suggestedLatency',b'\x00\x00\x07\x11hostApiSpecificStreamInfo'),(b'\x00\x00\x00\x85\x00\x00\x00\x02PaWasapiStreamInfo',b'\x00\x00\x2B\x11size',b'\x00\x00\x1C\x11hostApiType',b'\x00\x00\x2B\x11version',b'\x00\x00\x2B\x11flags',b'\x00\x00\x2B\x11channelMask',b'\x00\x00\x8C\x11hostProcessorOutput',b'\x00\x00\x8C\x11hostProcessorInput',b'\x00\x00\x87\x11threadPriority',b'\x00\x00\x84\x11streamCategory',b'\x00\x00\x86\x11streamOption')), - _enums = (b'\x00\x00\x00\x78\x00\x00\x00\x15PaErrorCode\x00paNoError,paNotInitialized,paUnanticipatedHostError,paInvalidChannelCount,paInvalidSampleRate,paInvalidDevice,paInvalidFlag,paSampleFormatNotSupported,paBadIODeviceCombination,paInsufficientMemory,paBufferTooBig,paBufferTooSmall,paNullCallback,paBadStreamPtr,paTimedOut,paInternalError,paDeviceUnavailable,paIncompatibleHostApiSpecificStreamInfo,paStreamIsStopped,paStreamIsNotStopped,paInputOverflowed,paOutputUnderflowed,paHostApiNotFound,paInvalidHostApi,paCanNotReadFromACallbackStream,paCanNotWriteToACallbackStream,paCanNotReadFromAnOutputOnlyStream,paCanNotWriteToAnInputOnlyStream,paIncompatibleStreamHostApi,paBadBufferPtr',b'\x00\x00\x00\x1C\x00\x00\x00\x16PaHostApiTypeId\x00paInDevelopment,paDirectSound,paMME,paASIO,paSoundManager,paCoreAudio,paOSS,paALSA,paAL,paBeOS,paWDMKS,paJACK,paWASAPI,paAudioScienceHPI',b'\x00\x00\x00\x7E\x00\x00\x00\x16PaStreamCallbackResult\x00paContinue,paComplete,paAbort',b'\x00\x00\x00\x83\x00\x00\x00\x16PaWasapiFlags\x00paWinWasapiExclusive,paWinWasapiRedirectHostProcessor,paWinWasapiUseChannelMask,paWinWasapiPolling,paWinWasapiThreadPriority,paWinWasapiExplicitSampleFormat,paWinWasapiAutoConvert',b'\x00\x00\x00\x84\x00\x00\x00\x16PaWasapiStreamCategory\x00eAudioCategoryOther,eAudioCategoryCommunications,eAudioCategoryAlerts,eAudioCategorySoundEffects,eAudioCategoryGameEffects,eAudioCategoryGameMedia,eAudioCategoryGameChat,eAudioCategorySpeech,eAudioCategoryMovie,eAudioCategoryMedia',b'\x00\x00\x00\x86\x00\x00\x00\x16PaWasapiStreamOption\x00eStreamOptionNone,eStreamOptionRaw,eStreamOptionMatchFormat',b'\x00\x00\x00\x87\x00\x00\x00\x16PaWasapiThreadPriority\x00eThreadPriorityNone,eThreadPriorityAudio,eThreadPriorityCapture,eThreadPriorityDistribution,eThreadPriorityGames,eThreadPriorityPlayback,eThreadPriorityProAudio,eThreadPriorityWindowManager'), - _typenames = (b'\x00\x00\x00\x75PaAsioStreamInfo',b'\x00\x00\x00\x01PaDeviceIndex',b'\x00\x00\x00\x77PaDeviceInfo',b'\x00\x00\x00\x01PaError',b'\x00\x00\x00\x78PaErrorCode',b'\x00\x00\x00\x01PaHostApiIndex',b'\x00\x00\x00\x7APaHostApiInfo',b'\x00\x00\x00\x1CPaHostApiTypeId',b'\x00\x00\x00\x7CPaHostErrorInfo',b'\x00\x00\x00\x7DPaMacCoreStreamInfo',b'\x00\x00\x00\x2BPaSampleFormat',b'\x00\x00\x00\x8DPaStream',b'\x00\x00\x00\x52PaStreamCallback',b'\x00\x00\x00\x2BPaStreamCallbackFlags',b'\x00\x00\x00\x7EPaStreamCallbackResult',b'\x00\x00\x00\x7FPaStreamCallbackTimeInfo',b'\x00\x00\x00\x6BPaStreamFinishedCallback',b'\x00\x00\x00\x2BPaStreamFlags',b'\x00\x00\x00\x81PaStreamInfo',b'\x00\x00\x00\x82PaStreamParameters',b'\x00\x00\x00\x21PaTime',b'\x00\x00\x00\x83PaWasapiFlags',b'\x00\x00\x00\x8CPaWasapiHostProcessorCallback',b'\x00\x00\x00\x84PaWasapiStreamCategory',b'\x00\x00\x00\x85PaWasapiStreamInfo',b'\x00\x00\x00\x86PaWasapiStreamOption',b'\x00\x00\x00\x87PaWasapiThreadPriority',b'\x00\x00\x00\x2BPaWinWaveFormatChannelMask',b'\x00\x00\x00\x8BSInt32'), -) - -try: - for libname in ('portaudio', 'bin\\libportaudio-2.dll', 'lib/libportaudio.dylib'): - libname = find_library(libname) - if libname is not None: break - else: raise OSError(translations['portaudio_not_found']) - - portaudiolib = ffi.dlopen(libname) -except OSError: - if platform.system() == 'Darwin': libname = 'libportaudio.dylib' - elif platform.system() == 'Windows': - if platform.machine().lower() in ('arm64', 'aarch64'): platform_suffix = 'arm64' - else: platform_suffix = platform.architecture()[0] - - libname = 'libportaudio' + platform_suffix + ('-asio.dll' if 'SD_ENABLE_ASIO' in os.environ else '.dll') - else: raise - - libdir = configs.get("portaudiolib", os.path.join("assets", "binary")) - libpath = os.path.join(libdir, libname) - - if not os.path.exists(libpath): - with open(os.path.join(libdir, "portaudiolib.bin"), "rb") as f: - model = pickle.load(f) - - with open(libpath, "wb") as w: - w.write(model[libname]) - - portaudiolib = ffi.dlopen(os.path.abspath(libpath)) - -initialized = 0 -last_callback = None -sampleformats = {'float32': portaudiolib.paFloat32, 'int32': portaudiolib.paInt32, 'int24': portaudiolib.paInt24, 'int16': portaudiolib.paInt16, 'int8': portaudiolib.paInt8, 'uint8': portaudiolib.paUInt8} - -def play(data, samplerate=None, mapping=None, blocking=False, loop=False, **kwargs): - ctx = CallbackContext(loop=loop) - ctx.frames = ctx.check_data(data, mapping, kwargs.get('device')) - - def callback(outdata, frames, time, status): - assert len(outdata) == frames - ctx.callback_enter(status, outdata) - ctx.write_outdata(outdata) - ctx.callback_exit() - - ctx.start_stream(OutputStream, samplerate, ctx.output_channels, ctx.output_dtype, callback, blocking, prime_output_buffers_using_stream_callback=False, **kwargs) - -def rec(frames=None, samplerate=None, channels=None, dtype=None, out=None, mapping=None, blocking=False, **kwargs): - ctx = CallbackContext() - out, ctx.frames = ctx.check_out(out, frames, channels, dtype, mapping) - - def callback(indata, frames, time, status): - assert len(indata) == frames - ctx.callback_enter(status, indata) - ctx.read_indata(indata) - ctx.callback_exit() - - ctx.start_stream(InputStream, samplerate, ctx.input_channels, - ctx.input_dtype, callback, blocking, **kwargs) - return out - -def playrec(data, samplerate=None, channels=None, dtype=None, out=None, input_mapping=None, output_mapping=None, blocking=False, **kwargs): - ctx = CallbackContext() - output_frames = ctx.check_data(data, output_mapping, kwargs.get('device')) - - if dtype is None: dtype = ctx.data.dtype - out, input_frames = ctx.check_out(out, output_frames, channels, dtype, input_mapping) - - if input_frames != output_frames: raise ValueError - ctx.frames = input_frames - - def callback(indata, outdata, frames, time, status): - assert len(indata) == len(outdata) == frames - ctx.callback_enter(status, indata) - ctx.read_indata(indata) - ctx.write_outdata(outdata) - ctx.callback_exit() - - ctx.start_stream(Stream, samplerate, (ctx.input_channels, ctx.output_channels), (ctx.input_dtype, ctx.output_dtype), callback, blocking, prime_output_buffers_using_stream_callback=False, **kwargs) - return out - -def wait(ignore_errors=True): - if last_callback: return last_callback.wait(ignore_errors) - -def stop(ignore_errors=True): - if last_callback: - last_callback.stream.stop(ignore_errors) - last_callback.stream.close(ignore_errors) - -def get_status(): - if last_callback: return last_callback.status - else: raise RuntimeError - -def get_stream(): - if last_callback: return last_callback.stream - else: raise RuntimeError - -def query_devices(device=None, kind=None): - if kind not in ('input', 'output', None): raise ValueError(f'{kind!r}') - if device is None and kind is None: return DeviceList(query_devices(i) for i in range(check(portaudiolib.Pa_GetDeviceCount()))) - - device = get_device_id(device, kind, raise_on_error=True) - info = portaudiolib.Pa_GetDeviceInfo(device) - - if not info: raise PortAudioError(device) - assert info.structVersion == 2 - name_bytes = ffi_string(info.name) - - try: - name = name_bytes.decode('utf-8') - except UnicodeDecodeError: - api_idx = portaudiolib.Pa_HostApiTypeIdToHostApiIndex - - if info.hostApi in (api_idx(portaudiolib.paDirectSound), api_idx(portaudiolib.paMME)): name = name_bytes.decode('mbcs') - elif info.hostApi == api_idx(portaudiolib.paASIO): name = name_bytes.decode(locale.getpreferredencoding()) - else: raise - - device_dict = { - 'name': name, - 'index': device, - 'hostapi': info.hostApi, - 'max_input_channels': info.maxInputChannels, - 'max_output_channels': info.maxOutputChannels, - 'default_low_input_latency': info.defaultLowInputLatency, - 'default_low_output_latency': info.defaultLowOutputLatency, - 'default_high_input_latency': info.defaultHighInputLatency, - 'default_high_output_latency': info.defaultHighOutputLatency, - 'default_samplerate': info.defaultSampleRate, - } - - if kind and device_dict['max_' + kind + '_channels'] < 1: raise ValueError(f"{kind} {device_dict['name']}") - return device_dict - -def query_hostapis(index=None): - if index is None: return tuple(query_hostapis(i) for i in range(check(portaudiolib.Pa_GetHostApiCount()))) - info = portaudiolib.Pa_GetHostApiInfo(index) - if not info: raise PortAudioError(index) - - assert info.structVersion == 1 - return { - 'name': ffi_string(info.name).decode(), - 'devices': [portaudiolib.Pa_HostApiDeviceIndexToDeviceIndex(index, i) for i in range(info.deviceCount)], - 'default_input_device': info.defaultInputDevice, - 'default_output_device': info.defaultOutputDevice, - } - -def check_input_settings(device=None, channels=None, dtype=None, extra_settings=None, samplerate=None): - parameters, dtype, _, samplerate = get_stream_parameters('input', device=device, channels=channels, dtype=dtype, latency=None, extra_settings=extra_settings, samplerate=samplerate) - check(portaudiolib.Pa_IsFormatSupported(parameters, ffi.NULL, samplerate)) - -def check_output_settings(device=None, channels=None, dtype=None, extra_settings=None, samplerate=None): - parameters, dtype, _, samplerate = get_stream_parameters('output', device=device, channels=channels, dtype=dtype, latency=None, extra_settings=extra_settings, samplerate=samplerate) - check(portaudiolib.Pa_IsFormatSupported(ffi.NULL, parameters, samplerate)) - -def sleep(msec): - portaudiolib.Pa_Sleep(msec) - -def get_portaudio_version(): - return portaudiolib.Pa_GetVersion(), ffi_string(portaudiolib.Pa_GetVersionText()).decode() - -class StreamBase: - def __init__(self, kind, samplerate=None, blocksize=None, device=None, channels=None, dtype=None, latency=None, extra_settings=None, callback=None, finished_callback=None, clip_off=None, dither_off=None, never_drop_input=None, prime_output_buffers_using_stream_callback=None, userdata=None, wrap_callback=None): - assert kind in ('input', 'output', 'duplex') - assert wrap_callback in ('array', 'buffer', None) - stream_flags = portaudiolib.paNoFlag - - if blocksize is None: blocksize = default.blocksize - if clip_off is None: clip_off = default.clip_off - if dither_off is None: dither_off = default.dither_off - if never_drop_input is None: never_drop_input = default.never_drop_input - if prime_output_buffers_using_stream_callback is None: prime_output_buffers_using_stream_callback = default.prime_output_buffers_using_stream_callback - if clip_off: stream_flags |= portaudiolib.paClipOff - if dither_off: stream_flags |= portaudiolib.paDitherOff - if never_drop_input: stream_flags |= portaudiolib.paNeverDropInput - if prime_output_buffers_using_stream_callback: stream_flags |= portaudiolib.paPrimeOutputBuffersUsingStreamCallback - - if kind == 'duplex': - idevice, odevice = split(device) - ichannels, ochannels = split(channels) - idtype, odtype = split(dtype) - ilatency, olatency = split(latency) - iextra, oextra = split(extra_settings) - - iparameters, idtype, isize, isamplerate = get_stream_parameters('input', idevice, ichannels, idtype, ilatency, iextra, samplerate) - oparameters, odtype, osize, osamplerate = get_stream_parameters('output', odevice, ochannels, odtype, olatency, oextra, samplerate) - - self._dtype = idtype, odtype - self._device = iparameters.device, oparameters.device - self._channels = iparameters.channelCount, oparameters.channelCount - self._samplesize = isize, osize - - if isamplerate != osamplerate: raise ValueError - else: samplerate = isamplerate - else: - parameters, self._dtype, self._samplesize, samplerate = get_stream_parameters(kind, device, channels, dtype, latency, extra_settings, samplerate) - self._device = parameters.device - self._channels = parameters.channelCount - iparameters = ffi.NULL - oparameters = ffi.NULL - - if kind == 'input': iparameters = parameters - elif kind == 'output': oparameters = parameters - - ffi_callback = ffi.callback('PaStreamCallback', error=portaudiolib.paAbort) - - if callback is None: callback_ptr = ffi.NULL - elif kind == 'input' and wrap_callback == 'buffer': - @ffi_callback - def callback_ptr(iptr, optr, frames, time, status, _): - data = mbuffer(iptr, frames, self._channels, self._samplesize) - return _wrap_callback(callback, data, frames, time, status) - elif kind == 'input' and wrap_callback == 'array': - @ffi_callback - def callback_ptr(iptr, optr, frames, time, status, _): - data = array(mbuffer(iptr, frames, self._channels, self._samplesize), self._channels, self._dtype) - return _wrap_callback(callback, data, frames, time, status) - elif kind == 'output' and wrap_callback == 'buffer': - @ffi_callback - def callback_ptr(iptr, optr, frames, time, status, _): - data = mbuffer(optr, frames, self._channels, self._samplesize) - return _wrap_callback(callback, data, frames, time, status) - elif kind == 'output' and wrap_callback == 'array': - @ffi_callback - def callback_ptr(iptr, optr, frames, time, status, _): - data = array(mbuffer(optr, frames, self._channels, self._samplesize), self._channels, self._dtype) - return _wrap_callback(callback, data, frames, time, status) - elif kind == 'duplex' and wrap_callback == 'buffer': - @ffi_callback - def callback_ptr(iptr, optr, frames, time, status, _): - idata, odata = mbuffer(iptr, frames, self._channels[0], self._samplesize[0]), mbuffer(optr, frames, self._channels[1], self._samplesize[1]) - return _wrap_callback(callback, idata, odata, frames, time, status) - elif kind == 'duplex' and wrap_callback == 'array': - @ffi_callback - def callback_ptr(iptr, optr, frames, time, status, _): - idata, odata = array(mbuffer(iptr, frames, self._channels[0], self._samplesize[0]), self._channels[0], self._dtype[0]), array(mbuffer(optr, frames, self._channels[1], self._samplesize[1]), self._channels[1], self._dtype[1]) - return _wrap_callback(callback, idata, odata, frames, time, status) - else: callback_ptr = ffi.cast('PaStreamCallback*', callback) - - self._callback = callback_ptr - if userdata is None: userdata = ffi.NULL - - self._ptr = ffi.new('PaStream**') - check(portaudiolib.Pa_OpenStream(self._ptr, iparameters, oparameters, samplerate, blocksize, stream_flags, callback_ptr, userdata), self.__class__.__name__) - - self._ptr = self._ptr[0] - self._blocksize = blocksize - - info = portaudiolib.Pa_GetStreamInfo(self._ptr) - if not info: raise PortAudioError - self._samplerate = info.sampleRate - - if not oparameters: self._latency = info.inputLatency - elif not iparameters: self._latency = info.outputLatency - else: self._latency = info.inputLatency, info.outputLatency - - if finished_callback: - if isinstance(finished_callback, ffi.CData): self._finished_callback = finished_callback - else: - def finished_callback_wrapper(_): - return finished_callback() - - self._finished_callback = ffi.callback('PaStreamFinishedCallback', finished_callback_wrapper) - - check(portaudiolib.Pa_SetStreamFinishedCallback(self._ptr, self._finished_callback)) - - _ptr = ffi.NULL - - @property - def samplerate(self): - return self._samplerate - - @property - def blocksize(self): - return self._blocksize - - @property - def device(self): - return self._device - - @property - def channels(self): - return self._channels - - @property - def dtype(self): - return self._dtype - - @property - def samplesize(self): - return self._samplesize - - @property - def latency(self): - return self._latency - - @property - def active(self): - if self.closed: return False - return check(portaudiolib.Pa_IsStreamActive(self._ptr)) == 1 - - @property - def stopped(self): - if self.closed: return True - return check(portaudiolib.Pa_IsStreamStopped(self._ptr)) == 1 - - @property - def closed(self): - return self._ptr == ffi.NULL - - @property - def time(self): - time = portaudiolib.Pa_GetStreamTime(self._ptr) - if not time: raise PortAudioError - return time - - @property - def cpu_load(self): - return portaudiolib.Pa_GetStreamCpuLoad(self._ptr) - - def __enter__(self): - self.start() - return self - - def __exit__(self, *args): - self.stop() - self.close() - - def start(self): - err = portaudiolib.Pa_StartStream(self._ptr) - if err != portaudiolib.paStreamIsNotStopped: check(err) - - def stop(self, ignore_errors=True): - err = portaudiolib.Pa_StopStream(self._ptr) - if not ignore_errors: check(err) - - def abort(self, ignore_errors=True): - err = portaudiolib.Pa_AbortStream(self._ptr) - if not ignore_errors: check(err) - - def close(self, ignore_errors=True): - err = portaudiolib.Pa_CloseStream(self._ptr) - self._ptr = ffi.NULL - if not ignore_errors: check(err) - -class InputStreamBase(StreamBase): - @property - def read_available(self): - return check(portaudiolib.Pa_GetStreamReadAvailable(self._ptr)) - - def _raw_read(self, frames): - channels, _ = split(self._channels) - samplesize, _ = split(self._samplesize) - - data = ffi.new('signed char[]', channels * samplesize * frames) - err = portaudiolib.Pa_ReadStream(self._ptr, data, frames) - - if err == portaudiolib.paInputOverflowed: overflowed = True - else: - check(err) - overflowed = False - - return ffi.mbuffer(data), overflowed - -class RawInputStream(InputStreamBase): - def __init__(self, samplerate=None, blocksize=None, device=None, channels=None, dtype=None, latency=None, extra_settings=None, callback=None, finished_callback=None, clip_off=None, dither_off=None, never_drop_input=None, prime_output_buffers_using_stream_callback=None): - StreamBase.__init__(self, kind='input', wrap_callback='buffer', **remove_self(locals())) - - read = InputStreamBase._raw_read - -class _OutputStreamBase(StreamBase): - @property - def write_available(self): - return check(portaudiolib.Pa_GetStreamWriteAvailable(self._ptr)) - - def _raw_write(self, data): - try: - data = ffi.from_buffer(data) - except AttributeError: - pass - except TypeError: - pass - - _, samplesize = split(self._samplesize) - _, channels = split(self._channels) - - samples, remainder = divmod(len(data), samplesize) - if remainder: raise ValueError - - frames, remainder = divmod(samples, channels) - if remainder: raise ValueError - - err = portaudiolib.Pa_WriteStream(self._ptr, data, frames) - - if err == portaudiolib.paOutputUnderflowed: underflowed = True - else: - check(err) - underflowed = False - - return underflowed - -class RawOutputStream(_OutputStreamBase): - def __init__(self, samplerate=None, blocksize=None, device=None, channels=None, dtype=None, latency=None, extra_settings=None, callback=None, finished_callback=None, clip_off=None, dither_off=None, never_drop_input=None, prime_output_buffers_using_stream_callback=None): - StreamBase.__init__(self, kind='output', wrap_callback='buffer', **remove_self(locals())) - - write = _OutputStreamBase._raw_write - -class RawStream(RawInputStream, RawOutputStream): - def __init__(self, samplerate=None, blocksize=None, device=None, channels=None, dtype=None, latency=None, extra_settings=None, callback=None, finished_callback=None, clip_off=None, dither_off=None, never_drop_input=None, prime_output_buffers_using_stream_callback=None): - StreamBase.__init__(self, kind='duplex', wrap_callback='buffer', **remove_self(locals())) - -class InputStream(InputStreamBase): - def __init__(self, samplerate=None, blocksize=None, device=None, channels=None, dtype=None, latency=None, extra_settings=None, callback=None, finished_callback=None, clip_off=None, dither_off=None, never_drop_input=None, prime_output_buffers_using_stream_callback=None): - StreamBase.__init__(self, kind='input', wrap_callback='array', **remove_self(locals())) - - def read(self, frames): - dtype, _ = split(self._dtype) - channels, _ = split(self._channels) - - data, overflowed = InputStreamBase._raw_read(self, frames) - data = array(data, channels, dtype) - - return data, overflowed - -class OutputStream(_OutputStreamBase): - def __init__(self, samplerate=None, blocksize=None, device=None, channels=None, dtype=None, latency=None, extra_settings=None, callback=None, finished_callback=None, clip_off=None, dither_off=None, never_drop_input=None, prime_output_buffers_using_stream_callback=None): - StreamBase.__init__(self, kind='output', wrap_callback='array', **remove_self(locals())) - - def write(self, data): - data = np.asarray(data) - _, dtype = split(self._dtype) - _, channels = split(self._channels) - - if data.ndim < 2: data = data.reshape(-1, 1) - elif data.ndim > 2: raise ValueError - - if data.shape[1] != channels: raise ValueError - if data.dtype != dtype: raise TypeError - if not data.flags.c_contiguous: raise TypeError - - return _OutputStreamBase._raw_write(self, data) - -class Stream(InputStream, OutputStream): - def __init__(self, samplerate=None, blocksize=None, device=None, channels=None, dtype=None, latency=None, extra_settings=None, callback=None, finished_callback=None, clip_off=None, dither_off=None, never_drop_input=None, prime_output_buffers_using_stream_callback=None): - StreamBase.__init__(self, kind='duplex', wrap_callback='array', **remove_self(locals())) - -class DeviceList(tuple): - __slots__ = () - - def __repr__(self): - idev = get_device_id(default.device['input'], 'input') - odev = get_device_id(default.device['output'], 'output') - digits = len(str(portaudiolib.Pa_GetDeviceCount() - 1)) - hostapi_names = [hostapi['name'] for hostapi in query_hostapis()] - - def get_mark(idx): - return (' ', '>', '<', '*')[(idx == idev) + 2 * (idx == odev)] - - text = '\n'.join('{mark} {idx:{dig}} {name}, {ha} ({ins} in, {outs} out)'.format(mark=get_mark(info['index']), idx=info['index'], dig=digits, name=info['name'], ha=hostapi_names[info['hostapi']], ins=info['max_input_channels'], outs=info['max_output_channels']) for info in self) - return text - -class CallbackFlags: - __slots__ = '_flags' - - def __init__(self, flags=0x0): - self._flags = flags - - def __repr__(self): - flags = str(self) - return f'' - - def __str__(self): - return ', '.join(name.replace('_', ' ') for name in dir(self) if not name.startswith('_') and getattr(self, name)) - - def __bool__(self): - return bool(self._flags) - - def __ior__(self, other): - if not isinstance(other, CallbackFlags): return NotImplemented - self._flags |= other._flags - return self - - @property - def input_underflow(self): - return self._hasflag(portaudiolib.paInputUnderflow) - - @input_underflow.setter - def input_underflow(self, value): - self._updateflag(portaudiolib.paInputUnderflow, value) - - @property - def input_overflow(self): - return self._hasflag(portaudiolib.paInputOverflow) - - @input_overflow.setter - def input_overflow(self, value): - self._updateflag(portaudiolib.paInputOverflow, value) - - @property - def output_underflow(self): - return self._hasflag(portaudiolib.paOutputUnderflow) - - @output_underflow.setter - def output_underflow(self, value): - self._updateflag(portaudiolib.paOutputUnderflow, value) - - @property - def output_overflow(self): - return self._hasflag(portaudiolib.paOutputOverflow) - - @output_overflow.setter - def output_overflow(self, value): - self._updateflag(portaudiolib.paOutputOverflow, value) - - @property - def priming_output(self): - return self._hasflag(portaudiolib.paPrimingOutput) - - def _hasflag(self, flag): - return bool(self._flags & flag) - - def _updateflag(self, flag, value): - if value: self._flags |= flag - else: self._flags &= ~flag - -class InputOutputPair: - _indexmapping = {'input': 0, 'output': 1} - - def __init__(self, parent, default_attr): - self._pair = [None, None] - self._parent = parent - self._default_attr = default_attr - - def __getitem__(self, index): - index = self._indexmapping.get(index, index) - value = self._pair[index] - if value is None: value = getattr(self._parent, self._default_attr)[index] - return value - - def __setitem__(self, index, value): - index = self._indexmapping.get(index, index) - self._pair[index] = value - - def __repr__(self): - return '[{0[0]!r}, {0[1]!r}]'.format(self) - -class default: - _pairs = 'device', 'channels', 'dtype', 'latency', 'extra_settings' - device = (None, None) - _default_channels = None, None - channels = _default_channels - _default_dtype = 'float32', 'float32' - dtype = _default_dtype - _default_latency = 'high', 'high' - latency = _default_latency - _default_extra_settings = None, None - extra_settings = _default_extra_settings - samplerate = None - blocksize = portaudiolib.paFramesPerBufferUnspecified - clip_off = False - dither_off = False - never_drop_input = False - prime_output_buffers_using_stream_callback = False - - def __init__(self): - for attr in self._pairs: - vars(self)[attr] = InputOutputPair(self, '_default_' + attr) - - def __setattr__(self, name, value): - if name in self._pairs: getattr(self, name)._pair[:] = split(value) - elif name in dir(self) and name != 'reset': object.__setattr__(self, name, value) - else: raise AttributeError(repr(name)) - - @property - def _default_device(self): - return (portaudiolib.Pa_GetDefaultInputDevice(), portaudiolib.Pa_GetDefaultOutputDevice()) - - @property - def hostapi(self): - return check(portaudiolib.Pa_GetDefaultHostApi()) - - def reset(self): - vars(self).clear() - self.__init__() - -if not hasattr(ffi, 'I_AM_FAKE'): - _default_instance = default() - default = _default_instance - -class PortAudioError(Exception): - def __init__(self, *args): - super().__init__(*args) - -class CallbackStop(Exception): - def __init__(self, *args): - super().__init__(*args) - - -class CallbackAbort(Exception): - def __init__(self, *args): - super().__init__(*args) - -class AsioSettings: - def __init__(self, channel_selectors): - if isinstance(channel_selectors, int): raise TypeError - self._selectors = ffi.new('int[]', channel_selectors) - self._streaminfo = ffi.new('PaAsioStreamInfo*', dict(size=ffi.sizeof('PaAsioStreamInfo'), hostApiType=portaudiolib.paASIO, version=1, flags=portaudiolib.paAsioUseChannelSelectors, channelSelectors=self._selectors)) - -class CoreAudioSettings: - def __init__(self, channel_map=None, change_device_parameters=False, fail_if_conversion_required=False, conversion_quality='max'): - conversion_dict = { - 'min': portaudiolib.paMacCoreConversionQualityMin, - 'low': portaudiolib.paMacCoreConversionQualityLow, - 'medium': portaudiolib.paMacCoreConversionQualityMedium, - 'high': portaudiolib.paMacCoreConversionQualityHigh, - 'max': portaudiolib.paMacCoreConversionQualityMax, - } - - if isinstance(channel_map, int): raise TypeError - - try: - self._flags = conversion_dict[conversion_quality.lower()] - except (KeyError, AttributeError): - raise ValueError(repr(list(conversion_dict))) - - if change_device_parameters: self._flags |= portaudiolib.paMacCoreChangeDeviceParameters - if fail_if_conversion_required: self._flags |= portaudiolib.paMacCoreFailIfConversionRequired - - self._streaminfo = ffi.new('PaMacCoreStreamInfo*') - portaudiolib.PaMacCore_SetupStreamInfo(self._streaminfo, self._flags) - - if channel_map is not None: - self._channel_map = ffi.new('SInt32[]', channel_map) - if len(self._channel_map) == 0: raise TypeError - portaudiolib.PaMacCore_SetupChannelMap(self._streaminfo, self._channel_map, len(self._channel_map)) - -class WasapiSettings: - def __init__(self, exclusive=False, auto_convert=False, explicit_sample_format=False): - flags = 0x0 - - if exclusive: flags |= portaudiolib.paWinWasapiExclusive - if auto_convert: flags |= portaudiolib.paWinWasapiAutoConvert - if explicit_sample_format: flags |= portaudiolib.paWinWasapiExplicitSampleFormat - - self._streaminfo = ffi.new('PaWasapiStreamInfo*', dict(size=ffi.sizeof('PaWasapiStreamInfo'), hostApiType=portaudiolib.paWASAPI, version=1, flags=flags,)) - -class CallbackContext: - frame = 0 - frames = 0 - input_channels = output_channels = None - input_dtype = output_dtype = None - silent_channels = None - - def __init__(self, loop=False): - self.loop = loop - self.event = threading.Event() - self.status = CallbackFlags() - - def check_data(self, data, mapping, device): - data = np.asarray(data) - - if data.ndim < 2: data = data.reshape(-1, 1) - elif data.ndim > 2: raise ValueError - - frames, channels = data.shape - dtype = check_dtype(data.dtype) - - mapping_is_explicit = mapping is not None - mapping, channels = check_mapping(mapping, channels) - - if data.shape[1] == 1: pass - elif data.shape[1] != len(mapping): raise ValueError - - if (mapping_is_explicit and np.array_equal(mapping, [0]) and query_devices(device, 'output')['max_output_channels'] >= 2): channels = 2 - - silent_channels = np.setdiff1d(np.arange(channels), mapping) - if len(mapping) + len(silent_channels) != channels: raise ValueError - - self.data = data - self.output_channels = channels - self.output_dtype = dtype - self.output_mapping = mapping - self.silent_channels = silent_channels - - return frames - - def check_out(self, out, frames, channels, dtype, mapping): - if out is None: - if frames is None: raise TypeError - if channels is None: channels = default.channels['input'] - - if channels is None: - if mapping is None: raise TypeError - else: channels = len(np.atleast_1d(mapping)) - - if dtype is None: dtype = default.dtype['input'] - - try: - out = np.empty((frames, channels), dtype, order='C') - except TypeError as e: - if not isinstance(frames, Integral): raise TypeError - if not isinstance(channels, Integral): raise TypeError - raise e - else: - frames, channels = out.shape - dtype = out.dtype - - dtype = check_dtype(dtype) - mapping, channels = check_mapping(mapping, channels) - if out.shape[1] != len(mapping): raise ValueError - - self.out = out - self.input_channels = channels - self.input_dtype = dtype - self.input_mapping = mapping - - return out, frames - - def callback_enter(self, status, data): - self.status |= status - self.blocksize = min(self.frames - self.frame, len(data)) - - def read_indata(self, indata): - for target, source in enumerate(self.input_mapping): - self.out[self.frame:self.frame + self.blocksize, target] = indata[:self.blocksize, source] - - def write_outdata(self, outdata): - outdata[:self.blocksize, self.output_mapping] = self.data[self.frame:self.frame + self.blocksize] - outdata[:self.blocksize, self.silent_channels] = 0 - - if self.loop and self.blocksize < len(outdata): - self.frame = 0 - outdata = outdata[self.blocksize:] - - self.blocksize = min(self.frames, len(outdata)) - self.write_outdata(outdata) - else: - outdata[self.blocksize:] = 0 - - def callback_exit(self): - if not self.blocksize: raise CallbackAbort - self.frame += self.blocksize - - def finished_callback(self): - self.event.set() - - with contextlib.suppress(AttributeError): - del self.data - - with contextlib.suppress(AttributeError): - del self.out - - self.stream._callback = None - self.stream._finished_callback = None - - def start_stream(self, StreamClass, samplerate, channels, dtype, callback, blocking, **kwargs): - stop() - - self.stream = StreamClass(samplerate=samplerate, channels=channels, dtype=dtype, callback=callback, finished_callback=self.finished_callback, **kwargs) - self.stream.start() - - global last_callback - last_callback = self - if blocking: self.wait() - - def wait(self, ignore_errors=True): - try: - self.event.wait() - finally: - self.stream.close(ignore_errors) - - return self.status if self.status else None - -def ffi_string(cdata): - return ffi.string(cdata) - -def remove_self(d): - d = d.copy() - del d['self'] - - return d - -def check_mapping(mapping, channels): - if mapping is None: mapping = np.arange(channels) - else: - mapping = np.array(mapping, copy=True) - mapping = np.atleast_1d(mapping) - - if mapping.min() < 1: raise ValueError - - channels = mapping.max() - mapping -= 1 - - return mapping, channels - -def check_dtype(dtype): - dtype = np.dtype(dtype).name - - if dtype in sampleformats: pass - elif dtype == 'float64': dtype = np.dtype(np.float32).name - else: raise TypeError - - return dtype - -def get_stream_parameters(kind, device, channels, dtype, latency, extra_settings, samplerate): - assert kind in ('input', 'output') - - if device is None: device = default.device - device = get_device_id(device, kind, raise_on_error=True) - - if channels is None: channels = default.channels - channels = select_input_or_output(channels, kind) - - if dtype is None: dtype = default.dtype - dtype = select_input_or_output(dtype, kind) - - if latency is None: latency = default.latency - latency = select_input_or_output(latency, kind) - - if extra_settings is None: extra_settings = default.extra_settings - extra_settings = select_input_or_output(extra_settings, kind) - - if samplerate is None: samplerate = default.samplerate - info = query_devices(device) - - if channels is None: channels = info['max_' + kind + '_channels'] - - try: - dtype = sys.modules['numpy'].dtype(dtype).name - except Exception: - pass - - try: - sampleformat = sampleformats[dtype] - except KeyError as e: - raise ValueError - - samplesize = check(portaudiolib.Pa_GetSampleSize(sampleformat)) - - if latency in ('low', 'high'): latency = info['default_' + latency + '_' + kind + '_latency'] - if samplerate is None: samplerate = info['default_samplerate'] - - parameters = ffi.new('PaStreamParameters*', (device, channels, sampleformat, latency, extra_settings._streaminfo if extra_settings else ffi.NULL)) - return parameters, dtype, samplesize, samplerate - -def _wrap_callback(callback, *args): - args = args[:-1] + (CallbackFlags(args[-1]),) - - try: - callback(*args) - except CallbackStop: - return portaudiolib.paComplete - except CallbackAbort: - return portaudiolib.paAbort - - return portaudiolib.paContinue - -def mbuffer(ptr, frames, channels, samplesize): - return ffi.buffer(ptr, frames * channels * samplesize) - -def array(buffer, channels, dtype): - data = np.frombuffer(buffer, dtype=dtype) - data.shape = -1, channels - return data - -def split(value): - if isinstance(value, (str, bytes)): return value, value - - try: - invalue, outvalue = value - except TypeError: - invalue = outvalue = value - except ValueError as e: - raise ValueError - - return invalue, outvalue - -def check(err, msg=''): - if err >= 0: return err - - errormsg = ffi_string(portaudiolib.Pa_GetErrorText(err)).decode() - if msg: errormsg = f'{msg}: {errormsg}' - - if err == portaudiolib.paUnanticipatedHostError: - info = portaudiolib.Pa_GetLastHostErrorInfo() - host_api = portaudiolib.Pa_HostApiTypeIdToHostApiIndex(info.hostApiType) - - hosterror_text = ffi_string(info.errorText).decode() - hosterror_info = host_api, info.errorCode, hosterror_text - - raise PortAudioError(errormsg, err, hosterror_info) - raise PortAudioError(errormsg, err) - -def select_input_or_output(value_or_pair, kind): - ivalue, ovalue = split(value_or_pair) - - if kind == 'input': return ivalue - elif kind == 'output': return ovalue - - assert False - -def get_device_id(id_or_query_string, kind, raise_on_error=False): - assert kind in ('input', 'output', None) - - if id_or_query_string is None: id_or_query_string = default.device - idev, odev = split(id_or_query_string) - - if kind == 'input': id_or_query_string = idev - elif kind == 'output': id_or_query_string = odev - else: - if idev == odev: id_or_query_string = idev - else: raise ValueError(id_or_query_string) - - if isinstance(id_or_query_string, int): return id_or_query_string - device_list, matches, exact_device_matches = [], [], [] - - for id, info in enumerate(query_devices()): - if not kind or info['max_' + kind + '_channels'] > 0: - hostapi_info = query_hostapis(info['hostapi']) - device_list.append((id, info['name'], hostapi_info['name'])) - - query_string = id_or_query_string.lower() - substrings = query_string.split() - - for id, device_string, hostapi_string in device_list: - full_string = device_string + ', ' + hostapi_string - pos = 0 - - for substring in substrings: - pos = full_string.lower().find(substring, pos) - if pos < 0: break - pos += len(substring) - else: - matches.append((id, full_string)) - if query_string in [device_string.lower(), full_string.lower()]: exact_device_matches.append(id) - - if kind is None: kind = 'input/output' - if not matches: - if raise_on_error: raise ValueError(repr(id_or_query_string)) - else: return -1 - - if len(matches) > 1: - if len(exact_device_matches) == 1: return exact_device_matches[0] - - if raise_on_error: raise ValueError(kind + " " + repr(id_or_query_string) + ':\n' + '\n'.join(f'[{id}] {name}' for id, name in matches)) - else: return -1 - - return matches[0][0] - -def initialize(): - old_stderr = None - - try: - old_stderr = os.dup(2) - devnull = os.open(os.devnull, os.O_WRONLY) - os.dup2(devnull, 2) - os.close(devnull) - except OSError: - pass - try: - check(portaudiolib.Pa_Initialize()) - global initialized - initialized += 1 - finally: - if old_stderr is not None: - os.dup2(old_stderr, 2) - os.close(old_stderr) - -def terminate(): - global initialized - - check(portaudiolib.Pa_Terminate()) - initialized -= 1 - - -def exit_handler(): - assert initialized >= 0 - - if last_callback: - last_callback.stream.stop() - last_callback.stream.close() - - while initialized: - terminate() - -atexit.register(exit_handler) -initialize() \ No newline at end of file diff --git a/main/library/audio/upscaler.py b/main/library/audio/upscaler.py deleted file mode 100644 index 42f68311ad52022a2862f7ac265d9e90f677cecd..0000000000000000000000000000000000000000 --- a/main/library/audio/upscaler.py +++ /dev/null @@ -1,1468 +0,0 @@ -import os -import sys -import math -import torch -import einops -import librosa - -import numpy as np -import torch.nn as nn -import torch.nn.functional as F -import torch.nn.utils.parametrize as parametrize - -from inspect import isfunction -from abc import abstractmethod -from torch.nn.utils import remove_weight_norm -from torch.nn.utils.parametrizations import weight_norm -from scipy.signal import butter, cheby1, cheby2, ellip, bessel, sosfiltfilt, resample_poly - -sys.path.append(os.getcwd()) - -from main.library.audio.features import mel - -def align_length(x, y): - Lx = len(x) - Ly = len(y) - - if Lx == Ly: return y - elif Lx > Ly: return np.pad(y, (0, Lx - Ly), mode="constant") - else: return y[:Lx] - -def subsampling(data, lowpass_ratio, fs_ori=44100, upsample_to_original=True): - assert len(data.shape) == 1 - - fs_down = int(lowpass_ratio * fs_ori) - y = resample_poly(data, fs_down, fs_ori) - - if upsample_to_original: - y = resample_poly(y, fs_ori, fs_down) - if len(y) != len(data): y = align_length(data, y) - - return y - -def lowpass_filter(x, highcutoff_freq, fs, order, ftype, upsample_to_original = True): - nyq = 0.5 * fs - hi = highcutoff_freq / nyq - - if ftype == "butter": sos = butter(order, hi, btype="low", output="sos") - elif ftype == "cheby1": sos = cheby1(order, 0.1, hi, btype="low", output="sos") - elif ftype == "cheby2": sos = cheby2(order, 60, hi, btype="low", output="sos") - elif ftype == "ellip": sos = ellip(order, 0.1, 60, hi, btype="low", output="sos") - elif ftype == "bessel": sos = bessel(order, hi, btype="low", output="sos") - else: raise ValueError - - y = sosfiltfilt(sos, x) - if len(y) != len(x): y = align_length(x, y) - - y = subsampling(y, lowpass_ratio=highcutoff_freq / int(fs / 2), fs_ori=fs, upsample_to_original=upsample_to_original) - return y - -def lowpass(audio, sr, filter_name, filter_order, cutoff_freq, upsample_to_original = True): - assert len(audio.shape) == 1 or (len(audio.shape) == 2 and (audio.shape[0] == 1 or audio.shape[0] == 2)) - if filter_name == "cheby": filter_name = "cheby1" - assert filter_order >= 2 and filter_order <= 10 - if cutoff_freq == sr: cutoff_freq -= 1 - - if len(audio.shape) == 2: - lowpassed_audio = np.zeros_like(audio) - - for i in range(audio.shape[0]): - lowpassed_audio[i] = lowpass_filter( - x=audio[i], - highcutoff_freq=int(cutoff_freq), - fs=sr, - order=filter_order, - ftype=filter_name, - upsample_to_original=upsample_to_original - ) - else: - lowpassed_audio = lowpass_filter( - x=audio, - highcutoff_freq=int(cutoff_freq), - fs=sr, - order=filter_order, - ftype=filter_name, - upsample_to_original=upsample_to_original - ) - - if upsample_to_original: assert lowpassed_audio.shape == audio.shape - return lowpassed_audio.copy() - -def find_cutoff(x, percentile=0.95): - percentile = x[-1] * percentile - - for i in range(1, x.shape[0]): - if x[-i] < percentile: return x.shape[0] - i - - return 0 - -def locate_cutoff_freq(stft, percentile=0.985): - return find_cutoff(stft, percentile) - -def find_cutoff_freq(audio): - device = audio.device - if not audio.device.type.startswith(("cpu", "cuda", "xpu")): audio = audio.cpu() - - stft_spec = torch.stft(audio, n_fft=2048, hop_length=480, win_length=2048, window=torch.hann_window(2048).to(audio.device), center=False, pad_mode="reflect", normalized=False, onesided=True, return_complex=True) - stft_spec = stft_spec[0].T.abs().sum(dim=0).cumsum(dim=0).float() - - cutoff_freq = (locate_cutoff_freq(stft_spec.to(device), percentile=0.983) / 1024) * 24000 - if cutoff_freq < 1000: cutoff_freq = 24000 - - return cutoff_freq - -def get_diffusers_output_type_name(ddpm_module): - output_type_dict = {"v_prediction": "v_prediction", "noise": "epsilon", "x_start": "sample"} - return output_type_dict[ddpm_module.model_output_type] - -def get_diffusers_scheduler_config(ddpm_module, scheduler_args): - config = {"num_train_timesteps": ddpm_module.timesteps, "trained_betas": ddpm_module.betas.to("cpu"), "prediction_type": get_diffusers_output_type_name(ddpm_module)} - config.update(scheduler_args) - - return config - -def freeze_param(model): - model = model.eval() - model.train = lambda self: self - - for param in model.parameters(): - param.requires_grad = False - - return model - -def timestep_embedding(timesteps, dim, max_period=10000, repeat_only=False): - if not repeat_only: - half = dim // 2 - freqs = (-math.log(max_period) * torch.arange(start=0, end=half, dtype=torch.float32) / half).exp().to(device=timesteps.device) - - args = timesteps[:, None].float() * freqs[None] - embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) - - if dim % 2: embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1) - else: embedding = einops.repeat(timesteps, "b -> b d", d=dim) - - return embedding - -def zero_module(module): - for p in module.parameters(): - p.detach().zero_() - return module - -def register_buffer(model, variable_name, value, dtype = torch.float32): - if type(value) != torch.Tensor: value = torch.tensor(value, dtype=dtype) - model.register_buffer(variable_name, value) - return getattr(model,variable_name) - -def checkpoint(func, inputs, params, flag): - return CheckpointFunction.apply(func, len(inputs), *(tuple(inputs) + tuple(params))) if flag else func(*inputs) - -def exists(x): - return x is not None - -def default(val, d): - if exists(val): return val - return d() if isfunction(d) else d - -def init_weights(m, mean=0.0, std=0.01): - if m.__class__.__name__.find("Conv") != -1: m.weight.data.normal_(mean, std) - -def get_padding(kernel_size, dilation=1): - return int((kernel_size * dilation - dilation) / 2) - -def kaiser_sinc_filter1d(cutoff, half_width, kernel_size): - even = kernel_size % 2 == 0 - half_size = kernel_size // 2 - delta_f = 4 * half_width - - A = 2.285 * (half_size - 1) * math.pi * delta_f + 7.95 - - if A > 50.0: - beta = 0.1102 * (A - 8.7) - elif A >= 21.0: - beta = 0.5842 * (A - 21) ** 0.4 + 0.07886 * (A - 21.0) - else: - beta = 0.0 - - window = torch.kaiser_window( - kernel_size, - beta=beta, - periodic=False - ) - - time = ( - torch.arange(-half_size, half_size) + 0.5 - ) if even else ( - torch.arange(kernel_size) - half_size - ) - - if cutoff == 0: - filter = torch.zeros_like(time) - else: - filter = 2 * cutoff * window * torch.sinc(2 * cutoff * time) - filter /= filter.sum() - - return filter.view(1, 1, kernel_size) - -class GroupNorm32(nn.GroupNorm): - def forward(self, x): - return F.group_norm( - x, - self.num_groups, - self.weight, - self.bias, - self.eps - ) - -class CheckpointFunction(torch.autograd.Function): - @staticmethod - def forward(ctx, run_function, length, *args): - ctx.run_function = run_function - ctx.input_tensors = list(args[:length]) - ctx.input_params = list(args[length:]) - - with torch.no_grad(): - output_tensors = ctx.run_function(*ctx.input_tensors) - - return output_tensors - - @staticmethod - def backward(ctx, *output_grads): - ctx.input_tensors = [x.detach().requires_grad_(True) for x in ctx.input_tensors] - with torch.enable_grad(): - shallow_copies = [x.view_as(x) for x in ctx.input_tensors] - output_tensors = ctx.run_function(*shallow_copies) - - input_grads = torch.autograd.grad(output_tensors, ctx.input_tensors + ctx.input_params, output_grads, allow_unused=True) - del ctx.input_tensors, ctx.input_params, output_tensors - - return (None, None) + input_grads - -class UtilAudioMelSpec: - def __init__(self, nfft, hop_size, sample_rate, mel_size, frequency_min, frequency_max, device): - self.nfft = nfft - self.hop_size = hop_size - self.sample_rate = sample_rate - self.mel_size = mel_size - self.frequency_min = frequency_min - self.frequency_max = frequency_max if frequency_max is not None else (sample_rate // 2) - self.hann_window = torch.hann_window(self.nfft) - self.mel_frequncies = librosa.mel_frequencies(n_mels=self.mel_size, fmin=self.frequency_min, fmax=self.frequency_max) - self.mel_basis_tensor = mel(sr=self.sample_rate, n_fft=self.nfft, n_mels=self.mel_size, fmin=self.frequency_min, fmax=self.frequency_max, device=device, dtype=torch.float32) - - def stft_torch(self, audio_torch): - assert(len(audio_torch.shape) <= 3) - if (len(audio_torch.shape) == 1): audio_torch = audio_torch.unsqueeze(0) - shape_is_three = len(audio_torch.shape) == 3 - - if shape_is_three: - batch_size, channels_num, segment_samples = audio_torch.shape - audio_torch = audio_torch.reshape(batch_size * channels_num, segment_samples) - - spec_dict = dict() - audio_torch = F.pad(audio_torch.unsqueeze(1), (int((self.nfft - self.hop_size) / 2), int((self.nfft - self.hop_size) / 2)), mode='reflect').squeeze(1) - - device = audio_torch.device - if not audio_torch.device.type.startswith(("cpu", "cuda", "xpu")): audio_torch = audio_torch.cpu() - - spec_dict['stft'] = torch.stft(audio_torch, self.nfft, hop_length=self.hop_size, window=self.hann_window.to(audio_torch.device), center=False, pad_mode='reflect', normalized=False, onesided=True, return_complex=True) - spec_dict['mag'] = spec_dict['stft'].abs().to(device) - spec_dict['angle'] = spec_dict['stft'].angle().to(device) - - if shape_is_three: - _, time_steps, freq_bins = spec_dict['stft'].shape - for feature_name in spec_dict: - spec_dict[feature_name] = spec_dict[feature_name].reshape(batch_size, channels_num, time_steps, freq_bins) - - return spec_dict - - def get_mel_spec(self, audio): - while len(audio.shape) < 2: audio = audio.unsqueeze(0) - return ((self.mel_basis_tensor @ self.stft_torch(audio)["mag"]).clamp(min=1e-5) * 1.0).log() - -class ResnetBlock(nn.Module): - def __init__(self, in_channels, out_channels=None, conv_shortcut=False, dropout=0.0, temb_channels=512): - super().__init__() - self.in_channels = in_channels - out_channels = in_channels if out_channels is None else out_channels - self.out_channels = out_channels - self.use_conv_shortcut = conv_shortcut - - self.norm1 = nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True) - self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1) - if temb_channels > 0: self.temb_proj = nn.Linear(temb_channels, out_channels) - - self.norm2 = nn.GroupNorm(num_groups=32, num_channels=out_channels, eps=1e-6, affine=True) - self.dropout = nn.Dropout(dropout) - self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1) - - if self.in_channels != self.out_channels: - if self.use_conv_shortcut: self.conv_shortcut = nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1) - else: self.nin_shortcut = nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=1, padding=0) - - def forward(self, x, temb): - h = x - - h = self.norm1(h) - h = self.conv1(h * h.sigmoid()) - - if temb is not None: h = h + self.temb_proj(temb * temb.sigmoid())[:, :, None, None] - - h = self.norm2(h) - h = self.conv2(self.dropout(h * h.sigmoid())) - - if self.in_channels != self.out_channels: x = self.conv_shortcut(x) if self.use_conv_shortcut else self.nin_shortcut(x) - return x + h - -class AttnBlock(nn.Module): - def __init__(self, in_channels): - super().__init__() - self.in_channels = in_channels - self.norm = nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True) - self.q = nn.Conv2d(in_channels, in_channels, kernel_size=1, stride=1, padding=0) - self.k = nn.Conv2d(in_channels, in_channels, kernel_size=1, stride=1, padding=0) - self.v = nn.Conv2d(in_channels, in_channels, kernel_size=1, stride=1, padding=0) - self.proj_out = nn.Conv2d(in_channels, in_channels, kernel_size=1, stride=1, padding=0) - - def forward(self, x): - h_ = x - h_ = self.norm(h_) - q = self.q(h_) - k = self.k(h_) - v = self.v(h_) - b, c, h, w = q.shape - - return x + self.proj_out( - torch.bmm( - v.reshape(b, c, h * w).contiguous(), - F.softmax( - torch.bmm( - q.reshape(b, c, h * w).contiguous().permute(0, 2, 1).contiguous(), - k.reshape(b, c, h * w).contiguous() - ).contiguous() * (int(c) ** (-0.5)), - dim=2 - ).permute(0, 2, 1).contiguous() - ).contiguous().reshape(b, c, h, w).contiguous() - ) - -class _Downsample(nn.Module): - def __init__(self, in_channels, with_conv): - super().__init__() - self.with_conv = with_conv - if self.with_conv: self.conv = nn.Conv2d(in_channels, in_channels, kernel_size=3, stride=2, padding=0) - - def forward(self, x): - return self.conv(F.pad(x, (0, 1, 0, 1), mode="constant", value=0)) if self.with_conv else F.avg_pool2d(x, kernel_size=2, stride=2) - -class _Upsample(nn.Module): - def __init__(self, in_channels, with_conv): - super().__init__() - self.with_conv = with_conv - if self.with_conv: self.conv = nn.Conv2d(in_channels, in_channels, kernel_size=3, stride=1, padding=1) - - def forward(self, x): - x = F.interpolate(x, scale_factor=2.0, mode="nearest") - if self.with_conv: x = self.conv(x) - return x - -class Encoder(nn.Module): - def __init__(self, ch, ch_mult=(1, 2, 4, 8), num_res_blocks=2, attn_resolutions=[], dropout=0.0, resamp_with_conv=True, in_channels=1, resolution=256, z_channels=16, double_z=True, attn_type="vanilla"): - super().__init__() - self.temb_ch = 0 - self.num_resolutions = len(ch_mult) - self.num_res_blocks = num_res_blocks - curr_res = resolution - in_ch_mult = (1,) + tuple(ch_mult) - self.in_ch_mult = in_ch_mult - - self.conv_in = nn.Conv2d(in_channels, ch, kernel_size=3, stride=1, padding=1) - self.down = nn.ModuleList() - - for i_level in range(self.num_resolutions): - block = nn.ModuleList() - attn = nn.ModuleList() - - block_in = ch * in_ch_mult[i_level] - block_out = ch * ch_mult[i_level] - - for _ in range(self.num_res_blocks): - block.append(ResnetBlock(in_channels=block_in, out_channels=block_out, temb_channels=self.temb_ch, dropout=dropout)) - block_in = block_out - if curr_res in attn_resolutions: attn.append(AttnBlock(block_in)) - - down = nn.Module() - down.block = block - down.attn = attn - - if i_level != self.num_resolutions - 1: - down.downsample = _Downsample(block_in, resamp_with_conv) - curr_res = curr_res // 2 - - self.down.append(down) - - self.mid = nn.Module() - self.mid.block_1 = ResnetBlock(in_channels=block_in, out_channels=block_in, temb_channels=self.temb_ch, dropout=dropout) - self.mid.attn_1 = AttnBlock(block_in) - self.mid.block_2 = ResnetBlock(in_channels=block_in, out_channels=block_in, temb_channels=self.temb_ch, dropout=dropout) - - self.norm_out = nn.GroupNorm(num_groups=32, num_channels=block_in, eps=1e-6, affine=True) - self.conv_out = nn.Conv2d(block_in, 2 * z_channels if double_z else z_channels, kernel_size=3, stride=1, padding=1) - - def forward(self, x): - temb = None - hs = [self.conv_in(x)] - - for i_level in range(self.num_resolutions): - for i_block in range(self.num_res_blocks): - h = self.down[i_level].block[i_block](hs[-1], temb) - if len(self.down[i_level].attn) > 0: h = self.down[i_level].attn[i_block](h) - hs.append(h) - - if i_level != self.num_resolutions - 1: hs.append(self.down[i_level].downsample(hs[-1])) - - h = self.norm_out(self.mid.block_2(self.mid.attn_1(self.mid.block_1(hs[-1], temb)), temb)) - return self.conv_out(h * h.sigmoid()) - -class Decoder(nn.Module): - def __init__(self, ch, out_ch, ch_mult=(1, 2, 4, 8), num_res_blocks=2, attn_resolutions=[], dropout=0.0, resamp_with_conv=True, resolution=256, z_channels=16, attn_type="vanilla"): - super().__init__() - self.temb_ch = 0 - self.num_resolutions = len(ch_mult) - self.num_res_blocks = num_res_blocks - block_in = ch * ch_mult[self.num_resolutions - 1] - curr_res = resolution // 2 ** (self.num_resolutions - 1) - - self.up = nn.ModuleList() - self.conv_in = nn.Conv2d(z_channels, block_in, kernel_size=3, stride=1, padding=1) - - self.mid = nn.Module() - self.mid.block_1 = ResnetBlock(in_channels=block_in, out_channels=block_in, temb_channels=self.temb_ch, dropout=dropout) - self.mid.attn_1 = AttnBlock(block_in) - self.mid.block_2 = ResnetBlock(in_channels=block_in, out_channels=block_in, temb_channels=self.temb_ch, dropout=dropout) - - for i_level in reversed(range(self.num_resolutions)): - block = nn.ModuleList() - attn = nn.ModuleList() - - block_out = ch * ch_mult[i_level] - - for _ in range(self.num_res_blocks + 1): - block.append(ResnetBlock(in_channels=block_in, out_channels=block_out, temb_channels=self.temb_ch, dropout=dropout)) - block_in = block_out - if curr_res in attn_resolutions: attn.append(AttnBlock(block_in)) - - up = nn.Module() - up.block = block - up.attn = attn - - if i_level != 0: - up.upsample = _Upsample(block_in, resamp_with_conv) - curr_res = curr_res * 2 - - self.up.insert(0, up) - - self.norm_out = nn.GroupNorm(num_groups=32, num_channels=block_in, eps=1e-6, affine=True) - self.conv_out = nn.Conv2d(block_in, out_ch, kernel_size=3, stride=1, padding=1) - - def forward(self, z): - self.last_z_shape = z.shape - temb = None - - h = self.mid.block_2(self.mid.attn_1(self.mid.block_1(self.conv_in(z), temb)), temb) - for i_level in reversed(range(self.num_resolutions)): - for i_block in range(self.num_res_blocks + 1): - h = self.up[i_level].block[i_block](h, temb) - if len(self.up[i_level].attn) > 0: h = self.up[i_level].attn[i_block](h) - - if i_level != 0: h = self.up[i_level].upsample(h) - - h = self.norm_out(h) - return self.conv_out(h * h.sigmoid()) - -class DiagonalGaussianDistribution(object): - def __init__(self, parameters): - self.parameters = parameters - self.mean, self.logvar = parameters.chunk(2, dim=1) - self.logvar = (self.logvar).clamp(-30.0, 20.0) - self.std = (0.5 * self.logvar).exp() - - def sample(self): - return self.mean + self.std * torch.randn(self.mean.shape).to(device=self.parameters.device) - -class AutoencoderKL(nn.Module): - def __init__(self, embed_dim=None): - super().__init__() - self.encoder = Encoder(ch=128, ch_mult=(1, 2, 4, 8), num_res_blocks=2, attn_resolutions=[], dropout=0.1, resamp_with_conv=True, in_channels=1, resolution=256, z_channels=16, double_z=True, attn_type="vanilla") - self.decoder = Decoder(ch=128, out_ch=1, ch_mult=(1, 2, 4, 8), num_res_blocks=2, attn_resolutions=[], dropout=0.1, resamp_with_conv=True, resolution=256, z_channels=16, attn_type="vanilla") - self.quant_conv = nn.Conv2d(2 * 16, 2 * embed_dim, 1) - self.post_quant_conv = nn.Conv2d(embed_dim, 16, 1) - self.embed_dim = embed_dim - - def encode(self, x): - return DiagonalGaussianDistribution(self.quant_conv(self.encoder(x))) - - def decode(self, z): - return self.decoder(self.post_quant_conv(z)) - -class TimestepBlock(nn.Module): - @abstractmethod - def forward(self, x, emb): - return None - -class GEGLU(nn.Module): - def __init__(self, dim_in, dim_out): - super().__init__() - self.proj = nn.Linear(dim_in, dim_out * 2) - - def forward(self, x): - x, gate = self.proj(x).chunk(2, dim=-1) - return x * F.gelu(gate) - -class FeedForward(nn.Module): - def __init__(self, dim, dim_out=None, mult=4, glu=False, dropout=0.0): - super().__init__() - inner_dim = int(dim * mult) - dim_out = default(dim_out, dim) - project_in = nn.Sequential(nn.Linear(dim, inner_dim), nn.GELU()) if not glu else GEGLU(dim, inner_dim) - self.net = nn.Sequential(project_in, nn.Dropout(dropout), nn.Linear(inner_dim, dim_out)) - - def forward(self, x): - return self.net(x) - -class CrossAttention(nn.Module): - def __init__(self, query_dim, context_dim=None, heads=8, dim_head=64, dropout=0.0): - super().__init__() - inner_dim = dim_head * heads - context_dim = default(context_dim, query_dim) - self.scale = dim_head ** -0.5 - self.heads = heads - self.to_q = nn.Linear(query_dim, inner_dim, bias=False) - self.to_k = nn.Linear(context_dim, inner_dim, bias=False) - self.to_v = nn.Linear(context_dim, inner_dim, bias=False) - self.to_out = nn.Sequential(nn.Linear(inner_dim, query_dim), nn.Dropout(dropout)) - - def forward(self, x, context=None, mask=None): - h = self.heads - q = self.to_q(x) - - context = default(context, x) - - k = self.to_k(context) - v = self.to_v(context) - - q, k, v = map(lambda t: einops.rearrange(t, "b n (h d) -> (b h) n d", h=h), (q, k, v)) - sim = torch.einsum("b i d, b j d -> b i j", q, k) * self.scale - - if exists(mask): - mask = einops.repeat(einops.rearrange(mask, "b ... -> b (...)"), "b j -> (b h) () j", h=h) - sim.masked_fill_(~(mask == 1), -torch.finfo(sim.dtype).max) - - return self.to_out(einops.rearrange(torch.einsum("b i j, b j d -> b i d", sim.softmax(dim=-1), v), "(b h) n d -> b n (h d)", h=h)) - -class BasicTransformerBlock(nn.Module): - def __init__(self, dim, n_heads, d_head, dropout=0.0, context_dim=None, gated_ff=True, checkpoint=True): - super().__init__() - self.attn1 = CrossAttention(query_dim=dim, heads=n_heads, dim_head=d_head, dropout=dropout) - self.ff = FeedForward(dim, dropout=dropout, glu=gated_ff) - self.attn2 = CrossAttention(query_dim=dim, context_dim=context_dim, heads=n_heads, dim_head=d_head, dropout=dropout) - self.norm1 = nn.LayerNorm(dim) - self.norm2 = nn.LayerNorm(dim) - self.norm3 = nn.LayerNorm(dim) - self.checkpoint = checkpoint - - def forward(self, x, context=None, mask=None): - return checkpoint(self._forward, (x,), self.parameters(), self.checkpoint) if context is None else checkpoint(self._forward, (x, context, mask), self.parameters(), self.checkpoint) - - def _forward(self, x, context=None, mask=None): - x = self.attn1(self.norm1(x)) + x - x = self.attn2(self.norm2(x), context=context, mask=mask) + x - return self.ff(self.norm3(x)) + x - -class SpatialTransformer(nn.Module): - def __init__(self, in_channels, n_heads, d_head, depth=1, dropout=0.0, context_dim=None): - super().__init__() - context_dim = context_dim - self.in_channels = in_channels - inner_dim = n_heads * d_head - - self.norm = nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True) - self.proj_in = nn.Conv2d(in_channels, inner_dim, kernel_size=1, stride=1, padding=0) - self.transformer_blocks = nn.ModuleList([BasicTransformerBlock(inner_dim, n_heads, d_head, dropout=dropout, context_dim=context_dim) for _ in range(depth)]) - self.proj_out = zero_module(nn.Conv2d(inner_dim, in_channels, kernel_size=1, stride=1, padding=0)) - - def forward(self, x, context=None, mask=None): - _, _, h, w = x.shape - - x_in = x - x = einops.rearrange(self.proj_in(self.norm(x)), "b c h w -> b (h w) c") - - for block in self.transformer_blocks: - x = block(x, context=context, mask=mask) - - x = self.proj_out(einops.rearrange(x, "b (h w) c -> b c h w", h=h, w=w)) - return x + x_in - -class TimestepEmbedSequential(nn.Sequential, TimestepBlock): - def forward(self, x, emb, context_list=None, mask_list=None): - spatial_transformer_id = 0 - context_list = [None] + context_list - mask_list = [None] + mask_list - - for layer in self: - if isinstance(layer, TimestepBlock): x = layer(x, emb) - elif isinstance(layer, SpatialTransformer): - context, mask = (None, None) if spatial_transformer_id >= len(context_list) else (context_list[spatial_transformer_id], mask_list[spatial_transformer_id]) - x = layer(x, context, mask=mask) - spatial_transformer_id += 1 - else: x = layer(x) - - return x - -class Downsample(nn.Module): - def __init__(self, channels, use_conv, dims=2, out_channels=None, padding=1): - super().__init__() - self.channels = channels - self.out_channels = out_channels or channels - self.use_conv = use_conv - self.dims = dims - stride = 2 if dims != 3 else (1, 2, 2) - - if use_conv: - conv_fn = nn.Conv2d if dims == 2 else nn.Conv3d - self.op = conv_fn(self.channels, self.out_channels, 3, stride=stride, padding=padding) - else: - assert self.channels == self.out_channels - avg_fn = nn.AvgPool2d if dims == 2 else nn.AvgPool3d - self.op = avg_fn(kernel_size=stride, stride=stride) - - def forward(self, x): - assert x.shape[1] == self.channels - return self.op(x) - -class Upsample(nn.Module): - def __init__(self, channels, use_conv, dims=2, out_channels=None, padding=1): - super().__init__() - self.channels = channels - self.out_channels = out_channels or channels - self.use_conv = use_conv - self.dims = dims - if use_conv: - conv_fn = nn.Conv2d if dims == 2 else nn.Conv3d - self.conv = conv_fn(self.channels, self.out_channels, 3, padding=padding) - - def forward(self, x): - assert x.shape[1] == self.channels - x = F.interpolate(x, (x.shape[2], x.shape[3] * 2, x.shape[4] * 2), mode="nearest") if self.dims == 3 else F.interpolate(x, scale_factor=2, mode="nearest") - if self.use_conv: x = self.conv(x) - - return x - -class ResBlock(TimestepBlock): - def __init__(self, channels, emb_channels, dropout, out_channels=None, use_conv=False, use_scale_shift_norm=False, dims=2, use_checkpoint=False, up=False, down=False): - super().__init__() - self.channels = channels - self.emb_channels = emb_channels - self.dropout = dropout - self.out_channels = out_channels or channels - self.use_conv = use_conv - self.use_checkpoint = use_checkpoint - self.use_scale_shift_norm = use_scale_shift_norm - - self.in_layers = nn.Sequential(GroupNorm32(32, channels), nn.SiLU(), nn.Conv2d(channels, self.out_channels, 3, padding=1)) - self.updown = up or down - - if up: self.h_upd, self.x_upd = Upsample(channels, False, dims), Upsample(channels, False, dims) - elif down: self.h_upd, self.x_upd = Downsample(channels, False, dims), Downsample(channels, False, dims) - else: self.h_upd = self.x_upd = nn.Identity() - - self.emb_layers = nn.Sequential(nn.SiLU(), nn.Linear(emb_channels, 2 * self.out_channels if use_scale_shift_norm else self.out_channels)) - self.out_layers = nn.Sequential(GroupNorm32(32, self.out_channels), nn.SiLU(), nn.Dropout(p=dropout), zero_module(nn.Conv2d(self.out_channels, self.out_channels, 3, padding=1))) - - if self.out_channels == channels: self.skip_connection = nn.Identity() - elif use_conv: self.skip_connection = nn.Conv2d(channels, self.out_channels, 3, padding=1) - else: self.skip_connection = nn.Conv2d(channels, self.out_channels, 1) - - def forward(self, x, emb): - return checkpoint(self._forward, (x, emb), self.parameters(), self.use_checkpoint) - - def _forward(self, x, emb): - if self.updown: - in_rest, in_conv = self.in_layers[:-1], self.in_layers[-1] - - h = self.h_upd(in_rest(x)) - x = self.x_upd(x) - h = in_conv(h) - else: h = self.in_layers(x) - - emb_out = self.emb_layers(emb).type(h.dtype) - while len(emb_out.shape) < len(h.shape): - emb_out = emb_out[..., None] - - if self.use_scale_shift_norm: - out_norm, out_rest = self.out_layers[0], self.out_layers[1:] - scale, shift = torch.chunk(emb_out, 2, dim=1) - h = out_rest(out_norm(h) * (1 + scale) + shift) - else: h = self.out_layers(h + emb_out) - - return self.skip_connection(x) + h - -class AudioSRUnet(nn.Module): - def __init__(self, image_size = 64, in_channels = 32, model_channels = 128, out_channels = 16, num_res_blocks = 2, attention_resolutions = [8, 4, 2], dropout=0, channel_mult=[1, 2, 3, 5], conv_resample=True, num_classes=None, extra_film_condition_dim=None, use_checkpoint=False, num_heads=-1, num_head_channels=32, num_heads_upsample=-1, use_scale_shift_norm=False, transformer_depth=1, context_dim=None, n_embed=None): - super().__init__() - - if num_heads_upsample == -1: num_heads_upsample = num_heads - if num_heads == -1: assert num_head_channels != -1 - if num_head_channels == -1: assert num_heads != -1 - - self.image_size = image_size - self.in_channels = in_channels - self.model_channels = model_channels - self.out_channels = out_channels - self.num_res_blocks = num_res_blocks - self.attention_resolutions = attention_resolutions - self.dropout = dropout - self.channel_mult = channel_mult - self.conv_resample = conv_resample - self.num_classes = num_classes - self.extra_film_condition_dim = extra_film_condition_dim - self.use_checkpoint = use_checkpoint - self.num_heads = num_heads - self.num_head_channels = num_head_channels - self.num_heads_upsample = num_heads_upsample - self.predict_codebook_ids = n_embed is not None - time_embed_dim = model_channels * 4 - self.use_extra_film_by_concat = self.extra_film_condition_dim is not None - self.time_embed = nn.Sequential(nn.Linear(model_channels, time_embed_dim), nn.SiLU(), nn.Linear(time_embed_dim, time_embed_dim)) - - if self.num_classes is not None: self.label_emb = nn.Embedding(num_classes, time_embed_dim) - if self.extra_film_condition_dim is not None: self.film_emb = nn.Linear(self.extra_film_condition_dim, time_embed_dim) - if context_dim is not None and not isinstance(context_dim, list): context_dim = [context_dim] - elif context_dim is None: context_dim = [None] - - self.input_blocks = nn.ModuleList([TimestepEmbedSequential(nn.Conv2d(in_channels, model_channels, 3, padding=1))]) - self._feature_size = model_channels - input_block_chans = [model_channels] - ch = model_channels - ds = 1 - - for level, mult in enumerate(channel_mult): - for _ in range(num_res_blocks): - layers = [ResBlock(ch, time_embed_dim if not self.use_extra_film_by_concat else (time_embed_dim * 2), dropout, out_channels=mult * model_channels, dims=2, use_checkpoint=use_checkpoint, use_scale_shift_norm=use_scale_shift_norm)] - ch = mult * model_channels - - if ds in attention_resolutions: - if num_head_channels == -1: dim_head = ch // num_heads - else: - num_heads = ch // num_head_channels - dim_head = num_head_channels - - dim_head = ch // num_heads - layers.append(SpatialTransformer(ch, num_heads, dim_head, depth=transformer_depth, context_dim=None)) - - for context_dim_id in range(len(context_dim)): - layers.append(SpatialTransformer(ch, num_heads, dim_head, depth=transformer_depth, context_dim=context_dim[context_dim_id])) - - self.input_blocks.append(TimestepEmbedSequential(*layers)) - self._feature_size += ch - input_block_chans.append(ch) - - if level != len(channel_mult) - 1: - out_ch = ch - self.input_blocks.append(TimestepEmbedSequential(Downsample(ch, conv_resample, dims=2, out_channels=out_ch))) - ch = out_ch - - input_block_chans.append(ch) - ds *= 2 - self._feature_size += ch - - if num_head_channels == -1: dim_head = ch // num_heads - else: - num_heads = ch // num_head_channels - dim_head = num_head_channels - - dim_head = ch // num_heads - middle_layers = [ResBlock(ch, time_embed_dim if not self.use_extra_film_by_concat else (time_embed_dim * 2), dropout, dims=2, use_checkpoint=use_checkpoint, use_scale_shift_norm=use_scale_shift_norm,)] - middle_layers.append(SpatialTransformer(ch, num_heads, dim_head, depth=transformer_depth, context_dim=None)) - - for context_dim_id in range(len(context_dim)): - middle_layers.append(SpatialTransformer(ch, num_heads, dim_head, depth=transformer_depth, context_dim=context_dim[context_dim_id])) - - middle_layers.append(ResBlock(ch, time_embed_dim if not self.use_extra_film_by_concat else (time_embed_dim * 2), dropout, dims=2, use_checkpoint=use_checkpoint, use_scale_shift_norm=use_scale_shift_norm)) - self.middle_block = TimestepEmbedSequential(*middle_layers) - - self._feature_size += ch - self.output_blocks = nn.ModuleList([]) - - for level, mult in list(enumerate(channel_mult))[::-1]: - for i in range(num_res_blocks + 1): - ich = input_block_chans.pop() - layers = [ResBlock(ch + ich, time_embed_dim if (not self.use_extra_film_by_concat) else time_embed_dim * 2, dropout, out_channels=model_channels * mult, dims=2, use_checkpoint=use_checkpoint, use_scale_shift_norm=use_scale_shift_norm)] - - ch = model_channels * mult - if ds in attention_resolutions: - if num_head_channels == -1: dim_head = ch // num_heads - else: - num_heads = ch // num_head_channels - dim_head = num_head_channels - - dim_head = ch // num_heads - layers.append(SpatialTransformer(ch, num_heads, dim_head, depth=transformer_depth, context_dim=None)) - - for context_dim_id in range(len(context_dim)): - layers.append(SpatialTransformer(ch, num_heads, dim_head, depth=transformer_depth, context_dim=context_dim[context_dim_id])) - - if level and i == num_res_blocks: - out_ch = ch - layers.append(Upsample(ch, conv_resample, dims=2, out_channels=out_ch)) - ds //= 2 - - self.output_blocks.append(TimestepEmbedSequential(*layers)) - self._feature_size += ch - - self.out = nn.Sequential(GroupNorm32(32, ch), nn.SiLU(), zero_module(nn.Conv2d(model_channels, out_channels, 3, padding=1))) - if self.predict_codebook_ids: self.id_predictor = nn.Sequential(GroupNorm32(32, ch), nn.Conv2d(model_channels, n_embed, 1)) - - def forward(self, x, timesteps=None, y=None, context_list=list(), context_attn_mask_list=list(), **kwargs): - x = torch.concat([x, y], dim=1) - y = None - - assert (y is not None) == ( - self.num_classes is not None or - self.extra_film_condition_dim is not None - ) - - hs = [] - t_emb = timestep_embedding(timesteps, self.model_channels, repeat_only=False).to(x.to(next(self.parameters()).dtype)) - - emb = self.time_embed(t_emb) - if self.use_extra_film_by_concat: emb = torch.cat([emb, self.film_emb(y)], dim=-1) - - h = x.to(next(self.parameters()).dtype) - for module in self.input_blocks: - h = module(h, emb, context_list, context_attn_mask_list) - hs.append(h) - - h = self.middle_block(h, emb, context_list, context_attn_mask_list) - for module in self.output_blocks: - h = module(torch.cat([h, hs.pop()], dim=1), emb, context_list, context_attn_mask_list) - - h = h.type(x.dtype) - return self.id_predictor(h) if self.predict_codebook_ids else self.out(h) - -class UpSample1d(nn.Module): - def __init__( - self, - ratio=2, - kernel_size=None - ): - super().__init__() - self.ratio = ratio - self.stride = ratio - - kernel_size = int(6 * ratio // 2) * 2 if kernel_size is None else kernel_size - - self.pad = kernel_size // ratio - 1 - self.pad_left = self.pad * self.stride + (kernel_size - self.stride) // 2 - self.pad_right = self.pad * self.stride + (kernel_size - self.stride + 1) // 2 - - filter = kaiser_sinc_filter1d( - cutoff=0.5 / ratio, - half_width=0.6 / ratio, - kernel_size=kernel_size - ) - self.register_buffer("filter", filter) - - def forward(self, x): - x = self.ratio * F.conv_transpose1d( - F.pad( - x, - (self.pad, self.pad), - mode="replicate" - ), - self.filter.expand(x.size(1), -1, -1), - stride=self.stride, - groups=x.size(1) - ) - - return x[..., self.pad_left : -self.pad_right] - -class LowPassFilter1d(nn.Module): - def __init__( - self, - cutoff=0.5, - half_width=0.6, - stride=1, - kernel_size=12 - ): - super().__init__() - if cutoff < -0.0 or cutoff > 0.5: - raise ValueError - - even = kernel_size % 2 == 0 - self.pad_left = kernel_size // 2 - int(even) - self.pad_right = kernel_size // 2 - self.stride = stride - - filter = kaiser_sinc_filter1d( - cutoff, - half_width, - kernel_size - ) - self.register_buffer("filter", filter) - - def forward(self, x): - return F.conv1d( - F.pad( - x, - (self.pad_left, self.pad_right), - mode="replicate" - ), - self.filter.expand(x.size(1), -1, -1), - stride=self.stride, - groups=x.size(1) - ) - -class DownSample1d(nn.Module): - def __init__( - self, - ratio=2, - kernel_size=None - ): - super().__init__() - self.lowpass = LowPassFilter1d( - cutoff=0.5 / ratio, - half_width=0.6 / ratio, - stride=ratio, - kernel_size=int(6 * ratio // 2) * 2 if kernel_size is None else kernel_size, - ) - - def forward(self, x): - return self.lowpass(x) - -class Activation1d(nn.Module): - def __init__(self, activation, up_ratio = 2, down_ratio = 2, up_kernel_size = 12, down_kernel_size = 12): - super().__init__() - self.up_ratio = up_ratio - self.down_ratio = down_ratio - self.act = activation - self.upsample = UpSample1d(up_ratio, up_kernel_size) - self.downsample = DownSample1d(down_ratio, down_kernel_size) - - def forward(self, x): - return self.downsample(self.act(self.upsample(x))) - -class SnakeBeta(nn.Module): - def __init__(self, in_features, alpha=1.0, alpha_trainable=True, alpha_logscale=False): - super(SnakeBeta, self).__init__() - self.in_features = in_features - self.alpha_logscale = alpha_logscale - - if self.alpha_logscale: - self.alpha = nn.Parameter(torch.zeros(in_features) * alpha) - self.beta = nn.Parameter(torch.zeros(in_features) * alpha) - else: - self.alpha = nn.Parameter(torch.ones(in_features) * alpha) - self.beta = nn.Parameter(torch.ones(in_features) * alpha) - - self.alpha.requires_grad = alpha_trainable - self.beta.requires_grad = alpha_trainable - self.no_div_by_zero = 0.000000001 - - def forward(self, x): - alpha = self.alpha.unsqueeze(0).unsqueeze(-1) - beta = self.beta.unsqueeze(0).unsqueeze(-1) - - if self.alpha_logscale: - alpha = alpha.exp() - beta = beta.exp() - - return x + (1.0 / (beta + self.no_div_by_zero)) * pow((x * alpha).sin(), 2) - -class AMPBlock1(nn.Module): - def __init__(self, channels, kernel_size=3, dilation=(1, 3, 5), snake_logscale = 'snakebeta'): - super(AMPBlock1, self).__init__() - self.convs1 = nn.ModuleList([weight_norm(nn.Conv1d(channels, channels, kernel_size, 1, dilation=dilation[0], padding=get_padding(kernel_size, dilation[0]))), weight_norm(nn.Conv1d(channels, channels, kernel_size, 1, dilation=dilation[1], padding=get_padding(kernel_size, dilation[1]))), weight_norm(nn.Conv1d(channels, channels, kernel_size, 1, dilation=dilation[2], padding=get_padding(kernel_size, dilation[2])))]) - self.convs1.apply(init_weights) - self.convs2 = nn.ModuleList([weight_norm(nn.Conv1d(channels, channels, kernel_size, 1, dilation=1, padding=get_padding(kernel_size, 1))), weight_norm(nn.Conv1d(channels, channels, kernel_size, 1, dilation=1, padding=get_padding(kernel_size, 1))), weight_norm(nn.Conv1d(channels, channels, kernel_size, 1, dilation=1, padding=get_padding(kernel_size, 1)))]) - self.convs2.apply(init_weights) - - self.num_layers = len(self.convs1) + len(self.convs2) - self.activations = nn.ModuleList([Activation1d(activation=SnakeBeta(channels, alpha_logscale=snake_logscale)) for _ in range(self.num_layers)]) - - def forward(self, x): - for c1, c2, a1, a2 in zip(self.convs1, self.convs2, self.activations[::2], self.activations[1::2]): - x = c2(a2(c1(a1(x)))) + x - - return x - - def remove_weight_norm(self): - for l in self.convs1: - if hasattr(l, "parametrizations") and "weight" in l.parametrizations: parametrize.remove_parametrizations(l, "weight", leave_parametrized=True) - else: remove_weight_norm(l) - for l in self.convs2: - if hasattr(l, "parametrizations") and "weight" in l.parametrizations: parametrize.remove_parametrizations(l, "weight", leave_parametrized=True) - else: remove_weight_norm(l) - -class SRVocoder(nn.Module): - def __init__(self, num_mels = 256, upsample_initial_channel = 1536, resblock_kernel_sizes = [3, 7, 11], resblock_dilation_sizes = [[1, 3, 5], [1, 3, 5], [1, 3, 5]], upsample_rates = [10, 6, 2, 2, 2], upsample_kernel_sizes = None, snake_logscale = True): - super(SRVocoder, self).__init__() - if upsample_kernel_sizes is None: upsample_kernel_sizes = [upsample_rate * 2 for upsample_rate in upsample_rates] - - self.audio_block = nn.ModuleDict() - self.audio_block["downsamples"] = nn.ModuleList() - self.audio_block["emb"] = nn.Conv1d(1, upsample_initial_channel // (2 ** len(upsample_rates)), 7, bias=True, padding=(7 - 1) // 2) - - for i in reversed(range(len(upsample_kernel_sizes))): - self.audio_block["downsamples"] += [nn.Sequential(nn.Conv1d(upsample_initial_channel // (2 ** (i + 1)), upsample_initial_channel // (2 ** i), upsample_kernel_sizes[i], upsample_rates[i], padding=upsample_rates[i] - (upsample_kernel_sizes[i] % 2 == 0), bias=True), nn.LeakyReLU(negative_slope = 0.1))] - - self.num_kernels = len(resblock_kernel_sizes) - self.num_upsamples = len(upsample_rates) - self.conv_pre = weight_norm(nn.Conv1d(num_mels, upsample_initial_channel, 7, 1, padding=3)) - - self.ups = nn.ModuleList() - for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)): - self.ups.append(nn.ModuleList([weight_norm(nn.ConvTranspose1d(upsample_initial_channel // (2 ** i), upsample_initial_channel // (2 ** (i + 1)), k, u, padding=(k - u) // 2))])) - - self.resblocks = nn.ModuleList() - for i in range(len(self.ups)): - ch = upsample_initial_channel // (2 ** (i + 1)) - for _, (k, d) in enumerate(zip(resblock_kernel_sizes, resblock_dilation_sizes)): - self.resblocks.append(AMPBlock1(ch, k, d)) - - activation_post = SnakeBeta(ch, alpha_logscale=snake_logscale) - self.activation_post = Activation1d(activation=activation_post) - self.conv_post = weight_norm(nn.Conv1d(ch, 1, 7, 1, padding=3)) - - self.conv_post.apply(init_weights) - for i in range(len(self.ups)): - self.ups[i].apply(init_weights) - - def forward(self, mel_spec, lr_audio): - audio_emb = self.audio_block["emb"](lr_audio.unsqueeze(1)) - audio_emb_list = [audio_emb] - - for i in range(self.num_upsamples - 1): - audio_emb = self.audio_block["downsamples"][i](audio_emb) - audio_emb_list += [audio_emb] - - x = self.conv_pre(mel_spec) - - for i in range(self.num_upsamples): - for i_up in range(len(self.ups[i])): - x = self.ups[i][i_up](x) + audio_emb_list[-1-i] - - xs = None - for j in range(self.num_kernels): - if xs is None: xs = self.resblocks[i * self.num_kernels + j](x) - else: xs += self.resblocks[i * self.num_kernels + j](x) - - x = xs / self.num_kernels - - return {'pred_hr_audio': self.conv_post(self.activation_post(x)).tanh().squeeze(1)} - - def remove_weight_norm(self): - if hasattr(self.conv2, "parametrizations") and "weight" in self.conv2.parametrizations: parametrize.remove_parametrizations(self.conv2, "weight", leave_parametrized=True) - else: remove_weight_norm(self.conv2) - - for l in self.ups: - for l_i in l: - if hasattr(l_i, "parametrizations") and "weight" in l_i.parametrizations: parametrize.remove_parametrizations(l_i, "weight", leave_parametrized=True) - else: remove_weight_norm(l_i) - - for l in self.resblocks: - l.remove_weight_norm() - - if hasattr(self.conv_pre, "parametrizations") and "weight" in self.conv_pre.parametrizations: parametrize.remove_parametrizations(self.conv_pre, "weight", leave_parametrized=True) - else: remove_weight_norm(self.conv_pre) - if hasattr(self.conv_post, "parametrizations") and "weight" in self.conv_post.parametrizations: parametrize.remove_parametrizations(self.conv_post, "weight", leave_parametrized=True) - else: remove_weight_norm(self.conv_post) - -class BetaSchedule: - @staticmethod - def linear(timesteps, start=1e-4, end=2e-2): - return np.linspace(start, end, timesteps) - - @staticmethod - def cosine(timesteps, s=0.008): - steps = timesteps + 1 - x = np.linspace(0, steps, steps) - - alphas_cumprod = np.cos(((x / steps) + s) / (1 + s) * np.pi * 0.5) ** 2 - alphas_cumprod = alphas_cumprod / alphas_cumprod[0] - - return np.clip(1 - (alphas_cumprod[1:] / alphas_cumprod[:-1]), a_min=0, a_max=0.999) - -class DDPM(nn.Module): - def __init__(self, model_output_type = 'noise', timesteps = 1000, loss_func = F.mse_loss, betas = None, beta_schedule_type = 'cosine', beta_arg_dict = dict(), unconditional_prob = 0, cfg_scale = None): - super().__init__() - self.model = AudioSRUnet() - self.model_output_type = model_output_type - self.loss_func = loss_func - self.timesteps = timesteps - self.set_noise_schedule(betas=betas, beta_schedule_type=beta_schedule_type, beta_arg_dict=beta_arg_dict, timesteps=timesteps) - self.unconditional_prob = unconditional_prob - self.cfg_scale = cfg_scale - - def set_noise_schedule(self, betas = None, beta_schedule_type = 'linear', beta_arg_dict = dict(), timesteps = 1000): - if betas is None: - beta_arg_dict.update({'timesteps':timesteps}) - betas = getattr(BetaSchedule, beta_schedule_type)(**beta_arg_dict) - - alphas = 1.0 - betas - alphas_cumprod = np.cumprod(alphas, axis=0) - alphas_cumprod_prev = np.append(1., alphas_cumprod[:-1]) - self.betas = register_buffer(model = self, variable_name = 'betas', value = betas) - self.alphas_cumprod = register_buffer(model = self, variable_name = 'alphas_cumprod', value = alphas_cumprod) - self.alphas_cumprod_prev = register_buffer(model = self, variable_name = 'alphas_cumprod_prev', value = alphas_cumprod_prev) - self.sqrt_alphas_cumprod = register_buffer(model = self, variable_name = 'sqrt_alphas_cumprod', value = np.sqrt(alphas_cumprod)) - self.sqrt_one_minus_alphas_cumprod = register_buffer(model = self, variable_name = 'sqrt_one_minus_alphas_cumprod', value = np.sqrt(1. - alphas_cumprod)) - self.log_one_minus_alphas_cumprod = register_buffer(model = self, variable_name = 'log_one_minus_alphas_cumprod', value = np.log(1. - alphas_cumprod)) - self.sqrt_recip_alphas_cumprod = register_buffer(model = self, variable_name = 'sqrt_recip_alphas_cumprod', value = np.sqrt(1. / alphas_cumprod)) - self.sqrt_recipm1_alphas_cumprod = register_buffer(model = self, variable_name = 'sqrt_recipm1_alphas_cumprod', value = np.sqrt(1. / alphas_cumprod - 1)) - posterior_variance = betas * (1. - alphas_cumprod_prev) / (1. - alphas_cumprod) - self.posterior_variance = register_buffer(model = self, variable_name = 'posterior_variance', value = posterior_variance) - self.posterior_log_variance_clipped = register_buffer(model = self, variable_name = 'posterior_log_variance_clipped', value = np.log(np.maximum(posterior_variance, 1e-20))) - self.posterior_mean_coef1 = register_buffer(model = self, variable_name = 'posterior_mean_coef1', value = betas * np.sqrt(alphas_cumprod_prev) / (1. - alphas_cumprod)) - self.posterior_mean_coef2 = register_buffer(model = self, variable_name = 'posterior_mean_coef2', value = (1. - alphas_cumprod_prev) * np.sqrt(alphas) / (1. - alphas_cumprod)) - - def apply_model(self, x, t, cond, is_cond_unpack, cfg_scale = None): - if cfg_scale is None or cfg_scale == 1.0: - if cond is None: return self.model(x, t) - elif is_cond_unpack: return self.model(x, t, **cond) - else: return self.model(x, t, cond) - else: - model_conditioned_output = self.model(x, t, **cond) if is_cond_unpack else self.model(x, t, cond) - unconditional_conditioning = self.get_unconditional_condition(cond=cond) - model_unconditioned_output = self.model(x, t, **unconditional_conditioning) if is_cond_unpack else self.model(x, t, unconditional_conditioning) - return model_unconditioned_output + cfg_scale * (model_conditioned_output - model_unconditioned_output) - - def get_unconditional_condition(self, cond = None, cond_shape = None, condition_device = None): - if cond_shape is None: cond_shape = cond.shape - if cond is not None and isinstance(cond, torch.Tensor): condition_device = cond.device - return (-11.4981 + torch.zeros(cond_shape)).to(condition_device) - -class DPMSolverMultistepScheduler: - def __init__( - self, - num_train_timesteps, - trained_betas, - prediction_type="epsilon", - solver_order=2, - **kwargs - ): - self.num_train_timesteps = num_train_timesteps - self.prediction_type = prediction_type - self.solver_order = solver_order - - betas = trained_betas.float() - - alphas = 1.0 - betas - alphas_cumprod = torch.cumprod(alphas, dim=0) - - self.alphas_cumprod = alphas_cumprod - self.alpha_t = alphas_cumprod.sqrt() - self.sigma_t = (1 - alphas_cumprod).sqrt() - - self.model_outputs = [] - self.timesteps = None - - def set_timesteps(self, num_inference_steps, device=None): - timesteps = torch.linspace( - self.num_train_timesteps - 1, - 0, - num_inference_steps, - dtype=torch.long, - device=device - ) - - self.timesteps = timesteps - self.model_outputs = [] - - def scale_model_input(self, sample, timestep): - return sample - - def convert_model_output(self, model_output, sample, timestep): - alpha_t = self.alpha_t[timestep].to(sample.device) - sigma_t = self.sigma_t[timestep].to(sample.device) - - while alpha_t.ndim < sample.ndim: - alpha_t = alpha_t.unsqueeze(-1) - sigma_t = sigma_t.unsqueeze(-1) - - if self.prediction_type == "epsilon": x0 = (sample - sigma_t * model_output) / alpha_t - elif self.prediction_type == "sample": x0 = model_output - elif self.prediction_type == "v_prediction": x0 = alpha_t * sample - sigma_t * model_output - else: raise ValueError - - return x0 - - def dpm_solver_first_order_update( - self, - model_output, - sample, - timestep, - prev_timestep - ): - alpha_t = self.alpha_t[timestep].to(sample.device) - sigma_t = self.sigma_t[timestep].to(sample.device) - - alpha_s = self.alpha_t[prev_timestep].to(sample.device) - sigma_s = self.sigma_t[prev_timestep].to(sample.device) - - while alpha_t.ndim < sample.ndim: - alpha_t = alpha_t.unsqueeze(-1) - sigma_t = sigma_t.unsqueeze(-1) - alpha_s = alpha_s.unsqueeze(-1) - sigma_s = sigma_s.unsqueeze(-1) - - return ( - alpha_s * model_output + - sigma_s * ( - sample - alpha_t * model_output - ) / sigma_t - ) - - def multistep_dpm_solver_second_order_update( - self, - model_output_list, - sample, - timestep, - prev_timestep - ): - m0 = model_output_list[-1] - m1 = model_output_list[-2] - - alpha_t = self.alpha_t[timestep] - sigma_t = self.sigma_t[timestep] - - alpha_s = self.alpha_t[prev_timestep] - sigma_s = self.sigma_t[prev_timestep] - - while alpha_t.ndim < sample.ndim: - alpha_t = alpha_t.unsqueeze(-1) - sigma_t = sigma_t.unsqueeze(-1) - alpha_s = alpha_s.unsqueeze(-1) - sigma_s = sigma_s.unsqueeze(-1) - - D0 = m0 - D1 = m0 - m1 - - x0 = ( - alpha_s * D0 + - sigma_s * ( - sample - alpha_t * D0 - ) / sigma_t - ) - - return x0 + 0.5 * D1 - - def step( - self, - model_output, - timestep, - sample, - return_dict=False - ): - step_index = (self.timesteps == timestep).nonzero()[0].item() - prev_timestep = 0 if (step_index == len(self.timesteps) - 1) else self.timesteps[step_index + 1] - - model_output = self.convert_model_output( - model_output, - sample, - timestep - ) - - self.model_outputs.append(model_output) - if len(self.model_outputs) > self.solver_order: self.model_outputs.pop(0) - - if len(self.model_outputs) == 1: - prev_sample = self.dpm_solver_first_order_update( - model_output, - sample, - timestep, - prev_timestep - ) - else: - prev_sample = self.multistep_dpm_solver_second_order_update( - self.model_outputs, - sample, - timestep, - prev_timestep - ) - - if return_dict: - return {"prev_sample": prev_sample} - - return (prev_sample,) - -class FlashSR: - def __init__(self, model_path, scale_factor_z = 0.3342, model_output_type = 'v_prediction', beta_schedule_type = 'cosine', device = None, is_half = False, **kwargs): - super().__init__() - self.scale_factor_z = scale_factor_z - self.device = device - self.is_half = is_half - - self.sr_vocoder = SRVocoder() - self.autoencoder = AutoencoderKL(embed_dim=16) - self.ddpm = DDPM(model_output_type=model_output_type, beta_schedule_type=beta_schedule_type, **kwargs) - self.util_mel_spec = UtilAudioMelSpec(nfft=2048, hop_size=480, sample_rate=48000, mel_size=256, frequency_min=20, frequency_max=24000, device=device) - - model = torch.load(model_path, map_location="cpu", weights_only=True) - - self.ddpm.load_state_dict(model["ldm"]) - self.sr_vocoder.load_state_dict(model["vocoder"]) - self.autoencoder.load_state_dict(model["vae"]) - self.autoencoder = freeze_param(self.autoencoder) - - self.ddpm.to(device).eval() - self.sr_vocoder.to(device).eval() - self.autoencoder.to(device).eval() - - self.ddpm.to(torch.float16 if is_half else torch.float32) - self.sr_vocoder.to(torch.float16 if is_half else torch.float32) - self.autoencoder.to(torch.float16 if is_half else torch.float32) - - def upscaler(self, lr_audio, num_steps = 1, lowpass_input = True, lowpass_cutoff_freq = None): - if lowpass_input: - if lowpass_cutoff_freq is None: lowpass_cutoff_freq = find_cutoff_freq(lr_audio) - - lr_audio = torch.from_numpy( - lowpass( - lr_audio.cpu().numpy(), - 48000, - filter_name='cheby', - filter_order=8, - cutoff_freq=lowpass_cutoff_freq - ) - ).to(lr_audio.device) - - with torch.no_grad(): - pred_hr_audio = self.infer( - lr_audio, - is_cond_unpack=False, - num_steps=num_steps, - device=lr_audio.device - ) - - return pred_hr_audio[...,:lr_audio.shape[-1]] - - def infer(self, cond, is_cond_unpack = False, num_steps = 20, scheduler_args = {'timestep_spacing': 'trailing'}, cfg_scale = None, device = None): - noise_scheduler = DPMSolverMultistepScheduler(**get_diffusers_scheduler_config(self.ddpm, scheduler_args)) - _, cond, additional_data_dict = self.preprocess(x_start=None, cond=cond) - - x_shape = cond.shape - noise_scheduler.set_timesteps(num_steps) - - x = torch.randn(x_shape, device=device) * 1.0 - - for t in noise_scheduler.timesteps: - denoiser_input = noise_scheduler.scale_model_input(x, t) - denoiser_input, cond = denoiser_input.to(torch.float16 if self.is_half else torch.float32), cond.to(torch.float16 if self.is_half else torch.float32) - - model_output = self.ddpm.apply_model(denoiser_input, torch.full((x_shape[0],), t, device=device, dtype=torch.long), cond, is_cond_unpack, cfg_scale=self.ddpm.cfg_scale if cfg_scale is None else cfg_scale) - x = noise_scheduler.step(model_output, t, x, return_dict=False)[0] - - return self.postprocess(x, additional_data_dict) - - def preprocess(self, x_start, cond = None): - x_dict = dict() - cond_dict = self.encode_to_z(cond) - - if x_start is not None: - x_dict = self.encode_to_z( - x_start, - scale_dict={ - 'mean_scale_factor': cond_dict['mean_scale_factor'], - 'var_scale_factor': cond_dict['var_scale_factor'] - } - ) - - return x_dict.get('z', None), cond_dict['z'], cond_dict - - def postprocess(self, x, additional_data_dict): - return self.denormalize_wav( - self.sr_vocoder( - self.z_to_mel(x).squeeze(1).transpose(1, 2), - additional_data_dict['norm_wav'] - )['pred_hr_audio'], - additional_data_dict - ) - - def denormalize_wav(self, waveform, scale_dict): - return ((waveform * 2.0) * (scale_dict['var_scale_factor'] + 1e-8)) + scale_dict['mean_scale_factor'] - - @torch.no_grad() - def encode_to_z(self, audio, normalize = True, scale_dict = None,): - assert len(audio.shape) == 2 - result_dict = {'wav': audio} - - if normalize: - audio, scale_dict = self.normalize_wav(audio, scale_dict=scale_dict) - result_dict['norm_wav'] = audio - result_dict.update(scale_dict) - - mel_spec = self.audio_to_mel(audio).to(torch.float16 if self.is_half else torch.float32) - result_dict['mel_spec'] = mel_spec - result_dict['z'] = self.autoencoder.encode(mel_spec).sample() * self.scale_factor_z - - return result_dict - - def normalize_wav(self, waveform, scale_dict): - mean_scale_factor = waveform.mean(dim=1, keepdim=True) if scale_dict is None else scale_dict['mean_scale_factor'] - waveform -= mean_scale_factor - - var_scale_factor = waveform.abs().max(dim=1, keepdim=True)[0] if scale_dict is None else scale_dict['var_scale_factor'] - waveform /= (var_scale_factor + 1e-8) - - return waveform * 0.5, {'mean_scale_factor': mean_scale_factor, 'var_scale_factor': var_scale_factor} - - @torch.no_grad() - def audio_to_mel(self, audio): - mel_spec = self.util_mel_spec.get_mel_spec(audio).to(self.device) - if len(mel_spec.shape) == 3: mel_spec = mel_spec.unsqueeze(1) - return mel_spec.permute(0, 1, 3, 2) - - def z_to_mel(self, z): - with torch.no_grad(): - x = (1.0 / self.scale_factor_z) * z - return self.autoencoder.decode(x.to(torch.float16 if self.is_half else torch.float32)) - -def get_window(window_size, fade_size): - window = torch.ones(window_size) - window[-fade_size:] *= torch.linspace(1, 0, fade_size) - window[:fade_size] *= torch.linspace(0, 1, fade_size) - return window - -def upscaler(audio, model, pbar = None, device = "cpu", C = 245760, step = 122880, fade_size = 24576, border = 122880): - if not torch.is_tensor(audio): audio = torch.from_numpy(audio.flatten()).to(device).float() - - if len(audio.shape) == 1: audio = audio.unsqueeze(0) - if audio.shape[1] > 2 * border and (border > 0): audio = F.pad(audio, (border, border), mode="reflect") - - result = torch.zeros((1,) + tuple(audio.shape), device=audio.device, dtype=torch.float32) - counter = torch.zeros((1,) + tuple(audio.shape), device=audio.device, dtype=torch.float32) - - i = 0 - pbar.total = pbar.total + math.ceil(audio.size(1) / step) - pbar.refresh() - - while i < audio.shape[1]: - part = audio[:, i : i + C] - length = part.shape[-1] - if length < C: part = F.pad(part, pad=(0, C - length), mode="reflect") if length > C // 2 + 1 else F.pad(part, pad=(0, C - length, 0, 0), mode="constant", value=0.0) - - out = model.upscaler(part, lowpass_input=True) - window = get_window(C, fade_size).to(audio.device) - - if i == 0: window[:fade_size] = 1 - elif i + C >= audio.shape[1]: window[-fade_size:] = 1 - - result[..., i : i + length] += out[..., :length] * window[..., :length] - counter[..., i : i + length] += window[..., :length] - - i += step - pbar.update(1) - - final_output = (result / counter).squeeze(0) - if audio.shape[1] > 2 * border and (border > 0): final_output = final_output[..., border:-border] - - return final_output \ No newline at end of file diff --git a/main/library/backends/directml.py b/main/library/backends/directml.py deleted file mode 100644 index 5dff3ebbf00bf114210d6d7c7624addb854d17fe..0000000000000000000000000000000000000000 --- a/main/library/backends/directml.py +++ /dev/null @@ -1,64 +0,0 @@ -import os -import gc -import sys -import torch -import subprocess - -sys.path.append(os.getcwd()) - -from main.library.embedders import fairseq -from main.library.backends.utils import GRU - -try: - import torch_directml -except: - torch_directml = None - -torch_available = torch_directml != None - -def device_count(): - return torch_directml.device_count() if torch_available else 0 - -def device_name(device_id = 0): - return torch_directml.device_name(device_id) if torch_available else "" - -def is_available(): - return torch_directml.is_available() if torch_available else False - -def empty_cache(): - empty_cache_path = os.path.join( - "main", - "library", - "backends", - "dml_empty_cache", - "empty_cache.exe" - ) - - if torch_available and os.path.exists(empty_cache_path): - subprocess.run([empty_cache_path], capture_output=True, text=True) - gc.collect() - -def forward_dml(ctx, x, scale): - ctx.scale = scale - res = x.clone().detach() - return res - -orig_conv_transpose1d = torch.nn.functional.conv_transpose1d - -def cpu_conv_transpose1d(input, weight, bias=None, stride=1, padding=0, output_padding=0, groups=1, dilation=1): - return orig_conv_transpose1d( - input.cpu(), - weight.cpu(), - bias.cpu() if bias is not None else bias, - stride, - padding, - output_padding, - groups, - dilation - ).to(input.device) - -if torch_available: - torch.nn.GRU = GRU - torch.inference_mode = torch.no_grad - fairseq.GradMultiply.forward = forward_dml - torch.nn.functional.conv_transpose1d = cpu_conv_transpose1d \ No newline at end of file diff --git a/main/library/backends/dml_empty_cache/empty_cache.exe b/main/library/backends/dml_empty_cache/empty_cache.exe deleted file mode 100644 index 1d2e2e1b434214fcdbe32e47f2adfd97a5d21687..0000000000000000000000000000000000000000 Binary files a/main/library/backends/dml_empty_cache/empty_cache.exe and /dev/null differ diff --git a/main/library/backends/dml_empty_cache/empty_cache.exp b/main/library/backends/dml_empty_cache/empty_cache.exp deleted file mode 100644 index 10fc0d553fa359ba3991141fd0c39f28ff4fec75..0000000000000000000000000000000000000000 Binary files a/main/library/backends/dml_empty_cache/empty_cache.exp and /dev/null differ diff --git a/main/library/backends/dml_empty_cache/empty_cache.lib b/main/library/backends/dml_empty_cache/empty_cache.lib deleted file mode 100644 index f100ca99732e72a8752437f802cdab08cc91684b..0000000000000000000000000000000000000000 Binary files a/main/library/backends/dml_empty_cache/empty_cache.lib and /dev/null differ diff --git a/main/library/backends/dml_empty_cache/empty_cache.obj b/main/library/backends/dml_empty_cache/empty_cache.obj deleted file mode 100644 index 511aec14b6ea8b7c8ef6bf3fc3a42a3d1aca1585..0000000000000000000000000000000000000000 Binary files a/main/library/backends/dml_empty_cache/empty_cache.obj and /dev/null differ diff --git a/main/library/backends/opencl.py b/main/library/backends/opencl.py deleted file mode 100644 index 53f6c8bfb107da1e1322a706dc4937300514b29a..0000000000000000000000000000000000000000 --- a/main/library/backends/opencl.py +++ /dev/null @@ -1,102 +0,0 @@ -import os -import sys -import torch -import platform -import subprocess - -try: - import pytorch_ocl -except: - pytorch_ocl = None - -sys.path.append(os.getcwd()) - -from main.library.backends.utils import GRU - -torch_available = pytorch_ocl != None -if torch_available: adaptive_orig = torch.nn.AdaptiveAvgPool2d - -def get_opencl_gpu_windows(): - gpus = "" - - try: - gpus = subprocess.check_output( - "wmic path win32_VideoController get name", - shell=True, - stderr=subprocess.DEVNULL - ) - except subprocess.CalledProcessError: - gpus = subprocess.check_output( - 'powershell "Get-CimInstance Win32_VideoController | Select-Object -ExpandProperty Name"', - shell=True, - stderr=subprocess.DEVNULL - ) - - return [ - gpu.strip() for gpu in gpus.decode().split('\n') - if gpu.strip() != "" - ] - -def get_opencl_gpu_linux(): - try: - return [ - gpu.strip() for gpu in subprocess.check_output("lspci | grep VGA", shell=True).decode().split('\n') - if gpu.strip() != "" - ] - except: - return [] - -def get_gpu_list(): - return ( - get_opencl_gpu_windows() if platform.system() == "Windows" else get_opencl_gpu_linux() - ) if torch_available else [] - -def device_count(): - return len(get_gpu_list()) if torch_available else 0 - -def device_name(device_id = 0): - return ( - get_gpu_list()[device_id] if device_id >= 0 and device_id < device_count() else "" - ) if torch_available else "" - -def is_available(): - return (device_count() > 0) if torch_available else False - -def group_norm(x, num_groups, weight=None, bias=None, eps=1e-5): - N, C = x.shape[:2] - assert C % num_groups == 0 - - shape = (N, num_groups, C // num_groups) + x.shape[2:] - x_reshaped = x.view(shape) - - dims = (2,) + tuple(range(3, x_reshaped.dim())) - mean = x_reshaped.mean(dim=dims, keepdim=True) - var = x_reshaped.var(dim=dims, keepdim=True, unbiased=False) - - x_norm = (x_reshaped - mean) / (var + eps).sqrt() - x_norm = x_norm.view_as(x) - - if weight is not None: - weight = weight.view(1, C, *([1] * (x.dim() - 2))) - x_norm = x_norm * weight - - if bias is not None: - bias = bias.view(1, C, *([1] * (x.dim() - 2))) - x_norm = x_norm + bias - - return x_norm - -def script(f, *_, **__): - f.graph = pytorch_ocl.torch._C.Graph() - return f - -def AdaptiveAvgPool2d(input): - input = input[0] if isinstance(input, tuple) else input - return adaptive_orig(input) - -if torch_available: - torch.nn.GRU = GRU - torch.jit.script = script - torch.inference_mode = torch.no_grad - torch.nn.functional.group_norm = group_norm - torch.nn.AdaptiveAvgPool2d = AdaptiveAvgPool2d \ No newline at end of file diff --git a/main/library/backends/utils.py b/main/library/backends/utils.py deleted file mode 100644 index 436f2791a47d982b6a216a9680db0eb4379a69fe..0000000000000000000000000000000000000000 --- a/main/library/backends/utils.py +++ /dev/null @@ -1,251 +0,0 @@ -import torch - -import numpy as np -import torch.nn.functional as F - -from librosa.util import pad_center -from scipy.signal import get_window - -class STFT(torch.nn.Module): - def __init__( - self, - filter_length=1024, - hop_length=512, - win_length=None, - window="hann", - pad_mode="reflect" - ): - super(STFT, self).__init__() - self.filter_length = filter_length - self.hop_length = hop_length - self.pad_amount = int(self.filter_length / 2) - self.cutoff = int(self.filter_length / 2 + 1) - self.win_length = win_length - self.pad_mode = pad_mode - - fourier_basis = np.fft.fft( - np.eye(self.filter_length) - ) - fourier_basis = np.vstack([ - np.real(fourier_basis[:self.cutoff, :]), - np.imag(fourier_basis[:self.cutoff, :]) - ]) - - forward_basis = torch.FloatTensor(fourier_basis) - inverse_basis = torch.FloatTensor(np.linalg.pinv(fourier_basis)) - - if win_length is None or not win_length: win_length = filter_length - assert filter_length >= win_length - - fft_window = torch.from_numpy( - pad_center( - get_window( - window, - win_length, - fftbins=True - ), - size=filter_length - ) - ).float() - - forward_basis *= fft_window - inverse_basis = (inverse_basis.T * fft_window).T - - self.register_buffer("forward_basis", forward_basis.float().unsqueeze(1)) - self.register_buffer("inverse_basis", inverse_basis.float()) - self.register_buffer("fft_window", fft_window.float()) - - def transform( - self, - input_data, - eps=1e-9, - return_phase=False, - center=True - ): - if center: - input_data = F.pad( - input_data, - (self.pad_amount, self.pad_amount), - mode=self.pad_mode - ) - - forward_transform = F.conv1d( - input_data.unsqueeze(1), - self.forward_basis, - stride=self.hop_length - ) - - real_part = forward_transform[:, :self.cutoff, :] - imag_part = forward_transform[:, self.cutoff:, :] - magnitude = (real_part**2 + imag_part**2 + eps).sqrt() - - if return_phase: - phase = imag_part.data.atan2(real_part.data) - return magnitude, phase - - return magnitude - - def inverse( - self, - magnitude, - phase - ): - cat = torch.cat([ - magnitude * phase.cos(), - magnitude * phase.sin() - ], dim=1) - - fold = torch.nn.Fold( - output_size=(1, (cat.size(-1) - 1) * self.hop_length + self.filter_length), - kernel_size=(1, self.filter_length), - stride=(1, self.hop_length) - ) - - inverse_transform = fold( - self.inverse_basis @ cat - )[:, 0, 0, self.pad_amount : -self.pad_amount] - - window_square_sum = ( - fold( - self.fft_window.cpu().pow(2).repeat(cat.size(-1), 1).T.unsqueeze(0) - ) - ) if str(cat.device).startswith("ocl") else ( - fold( - self.fft_window.pow(2).repeat(cat.size(-1), 1).T.unsqueeze(0) - ) - )[:, 0, 0, self.pad_amount : -self.pad_amount].to(cat.device) - - return inverse_transform / window_square_sum - -class GRU(torch.nn.RNNBase): - def __init__( - self, - input_size, - hidden_size, - num_layers=1, - bias=True, - batch_first=True, - dropout=0.0, - bidirectional=False, - device=None, - dtype=None - ): - super().__init__( - "GRU", - input_size=input_size, - hidden_size=hidden_size, - num_layers=num_layers, - bias=bias, - batch_first=batch_first, - dropout=dropout, - bidirectional=bidirectional, - device=device, - dtype=dtype - ) - - def _gru_layer(self, x, hx, weights): - weight_ih, weight_hh, bias_ih, bias_hh = weights - B, T, _ = x.shape - - gate_x = F.linear(x, weight_ih, bias_ih) - outputs = torch.empty(B, T, self.hidden_size, device=x.device, dtype=x.dtype) - - if bias_hh is not None: b_hr, b_hz, b_hn = bias_hh.chunk(3) - else: b_hr = b_hz = b_hn = None - - w_hr, w_hz, w_hn = weight_hh.chunk(3, dim=0) - - for t in range(T): - x_r, x_z, x_n = gate_x[:, t].chunk(3, dim=1) - r = (x_r + F.linear(hx, w_hr, b_hr)).sigmoid() - z = (x_z + F.linear(hx, w_hz, b_hz)).sigmoid() - - n = (x_n + r * F.linear(hx, w_hn, b_hn)).tanh() - hx = n + z * (hx - n) - outputs[:, t] = hx - - return outputs, hx - - def _gru(self, x, hx): - if not self.batch_first: x = x.permute(1, 0, 2) - num_directions = 2 if self.bidirectional else 1 - - h_n = [] - output_fwd, output_bwd = x, x - - for layer in range(self.num_layers): - fwd_idx = layer * num_directions - bwd_idx = fwd_idx + 1 if self.bidirectional else None - - weights_fwd = self._get_weights(fwd_idx) - h_fwd = hx[fwd_idx] - - out_fwd, h_out_fwd = self._gru_layer( - output_fwd, - h_fwd, - weights_fwd - ) - - h_n.append(h_out_fwd) - - if self.bidirectional: - weights_bwd = self._get_weights(bwd_idx) - h_bwd = hx[bwd_idx] - reversed_input = output_bwd.flip(dims=[1]) - - out_bwd, h_out_bwd = self._gru_layer( - reversed_input, - h_bwd, - weights_bwd - ) - - out_bwd = out_bwd.flip(dims=[1]) - h_n.append(h_out_bwd) - - output_fwd = torch.cat([out_fwd, out_bwd], dim=2) - output_bwd = output_fwd - else: output_fwd = out_fwd - - if layer < self.num_layers - 1 and self.dropout > 0: - output_fwd = F.dropout( - output_fwd, - p=self.dropout, - training=self.training - ) - - if self.bidirectional: output_bwd = output_fwd - - output = output_fwd - h_n = torch.stack(h_n, dim=0) - - if not self.batch_first: output = output.permute(1, 0, 2) - return output, h_n - - def _get_weights(self, layer_idx): - weights = self._all_weights[layer_idx] - - weight_ih = getattr(self, weights[0]) - weight_hh = getattr(self, weights[1]) - - bias_ih = getattr(self, weights[2]) if self.bias else None - bias_hh = getattr(self, weights[3]) if self.bias else None - - return weight_ih, weight_hh, bias_ih, bias_hh - - def forward(self, input, hx=None): - if input.dim() != 3: raise ValueError - - batch_size = input.size(0) if self.batch_first else input.size(1) - num_directions = 2 if self.bidirectional else 1 - - if hx is None: - hx = torch.zeros( - self.num_layers * num_directions, - batch_size, - self.hidden_size, - dtype=input.dtype, - device=input.device - ) - - self.check_forward_args(input, hx, batch_sizes=None) - return self._gru(input, hx) \ No newline at end of file diff --git a/main/library/backends/xpu.py b/main/library/backends/xpu.py deleted file mode 100644 index 66f632a0dfd424248c424e310d314890d2e19e9d..0000000000000000000000000000000000000000 --- a/main/library/backends/xpu.py +++ /dev/null @@ -1,92 +0,0 @@ -import torch - -from collections import defaultdict -from torch.amp.grad_scaler import _MultiDeviceReplicator, OptState, _refresh_per_optimizer_state - -def _amp_update_scale_(input, growth_tracker, found_inf, growth_factor=2.0, backoff_factor=0.5, growth_interval=2000): - if found_inf > 0: - input.mul_(backoff_factor) - growth_tracker.zero_() - else: - growth_tracker.add_(1) - - if growth_tracker >= growth_interval: - input.mul_(growth_factor) - growth_tracker.zero_() - - return input - -def _unscale_grads_(self, optimizer, inv_scale, found_inf, allow_fp16): - per_device_inv_scale = _MultiDeviceReplicator(inv_scale) - per_device_found_inf = _MultiDeviceReplicator(found_inf) - per_device_and_dtype_grads: dict[torch.device, dict[torch.dtype, list[torch.Tensor]]] = defaultdict(lambda: defaultdict(list)) - - with torch.no_grad(): - for group in optimizer.param_groups: - for param in group["params"]: - if not isinstance(param, torch.Tensor): raise AssertionError - if param.grad is None: continue - if (not allow_fp16) and param.grad.dtype == torch.float16: raise ValueError - - if param.grad.is_sparse: - if param.grad.dtype is torch.float16: param.grad = param.grad.coalesce() - to_unscale = param.grad._values() - else: to_unscale = param.grad - - per_device_and_dtype_grads[to_unscale.device][to_unscale.dtype].append(to_unscale) - - for device, per_dtype_grads in per_device_and_dtype_grads.items(): - for grads in per_dtype_grads.values(): - torch._amp_foreach_non_finite_check_and_unscale_(grads, per_device_found_inf.get(device), per_device_inv_scale.get(device)) - - return per_device_found_inf._per_device_tensors - -def unscale_(self, optimizer): - if not self._enabled: return - - self._check_scale_growth_tracker("unscale_") - optimizer_state = self._per_optimizer_states[id(optimizer)] - - if optimizer_state["stage"] is OptState.UNSCALED: raise RuntimeError - elif optimizer_state["stage"] is OptState.STEPPED: raise RuntimeError - - if self._scale is None: raise AssertionError - - inv_scale = self._scale.reciprocal() - found_inf = torch.full((), 0.0, dtype=torch.float32, device=self._scale.device) - - optimizer_state["found_inf_per_device"] = self._unscale_grads_(optimizer, inv_scale, found_inf, False) - optimizer_state["stage"] = OptState.UNSCALED - -def update(self, new_scale = None): - if not self._enabled: return - - _scale, _growth_tracker = self._check_scale_growth_tracker("update") - - if new_scale is not None: - if self._scale is None: raise AssertionError - - if isinstance(new_scale, float): self._scale.fill_(new_scale) - else: - if new_scale.device.type != self._device: raise AssertionError - if new_scale.numel() != 1: raise AssertionError - if new_scale.requires_grad is True: raise AssertionError - - self._scale.copy_(new_scale) - else: - found_infs = [found_inf.to(device=_scale.device, non_blocking=True) for state in self._per_optimizer_states.values() for found_inf in state["found_inf_per_device"].values()] - if len(found_infs) == 0: raise AssertionError - - found_inf_combined = found_infs[0] - if len(found_infs) > 1: - for i in range(1, len(found_infs)): - found_inf_combined += found_infs[i] - - _amp_update_scale_(_scale, _growth_tracker, found_inf_combined, self._growth_factor, self._backoff_factor, self._growth_interval) - - self._per_optimizer_states = defaultdict(_refresh_per_optimizer_state) - -def setup_gradscaler(): - torch.amp.grad_scaler.GradScaler._unscale_grads_ = _unscale_grads_ - torch.amp.grad_scaler.GradScaler.unscale_ = unscale_ - torch.amp.grad_scaler.GradScaler.update = update \ No newline at end of file diff --git a/main/library/backends/zluda.py b/main/library/backends/zluda.py deleted file mode 100644 index 3f7285aebc3ddefff7a7ae896683121676d9bc05..0000000000000000000000000000000000000000 --- a/main/library/backends/zluda.py +++ /dev/null @@ -1,27 +0,0 @@ -import torch - -def init_zluda(): - _torch_stft = torch.stft - _torch_istft = torch.istft - - def z_stft(input, window, *args, **kwargs): - return _torch_stft( - input=input.cpu(), window=window.cpu(), *args, **kwargs - ).to(input.device) - - def z_istft(input, window, *args, **kwargs): - return _torch_istft( - input=input.cpu(), window=window.cpu(), *args, **kwargs - ).to(input.device) - - def z_jit(f, *_, **__): - f.graph = torch._C.Graph() - return f - - torch.stft = z_stft - torch.istft = z_istft - torch.jit.script = z_jit - torch.backends.cudnn.enabled = False - torch.backends.cuda.enable_flash_sdp(False) - torch.backends.cuda.enable_math_sdp(True) - torch.backends.cuda.enable_mem_efficient_sdp(False) \ No newline at end of file diff --git a/main/library/embedders/fairseq.py b/main/library/embedders/fairseq.py deleted file mode 100644 index 6ceb9bdf01505dab6eaddbcba49767c1d7440d8e..0000000000000000000000000000000000000000 --- a/main/library/embedders/fairseq.py +++ /dev/null @@ -1,1999 +0,0 @@ -import os -import re -import sys -import math -import uuid -import torch -import types -import contextlib - -import numpy as np -import torch.nn.functional as F -import torch.nn.utils.parametrize as parametrize - -from torch import nn -from omegaconf import DictConfig, open_dict - -sys.path.append(os.getcwd()) - -def load_model(filename): - try: - state = torch.load(filename, map_location="cpu", weights_only=True) - except: - from main.app.variables import logger, configs, translations - - logger.warning(translations["hubert_warn_1"]) - logger.warning(translations["hubert_warn_2"]) - - if configs.get("allow_unsafe_fairseq", False): - class Dictionary: - def __init__(self, *args, **kwargs): - pass - - fairseq = types.ModuleType("fairseq") - fairseq_data = types.ModuleType("fairseq.data") - fairseq_data_dictionary = types.ModuleType("fairseq.data.dictionary") - fairseq_data_dictionary.Dictionary = Dictionary - fairseq.data = fairseq_data - fairseq_data.dictionary = fairseq_data_dictionary - sys.modules["fairseq"] = fairseq - sys.modules["fairseq.data"] = fairseq_data - sys.modules["fairseq.data.dictionary"] = fairseq_data_dictionary - - state = torch.load(filename, map_location="cpu", weights_only=False) - else: - sys.exit(0) - - model = HubertModel( - HubertConfig( - **state['cfg']['model'] - ), - num_classes=int(state['model']['label_embs_concat'].shape[0]) - ) - model.load_state_dict(state['model'], strict=False) - - return model - -def softmax(x, dim, onnx_trace = False): - return F.softmax(x.float(), dim=dim) if onnx_trace else F.softmax(x, dim=dim, dtype=torch.float32) - -def log_softmax(x, dim, onnx_trace = False): - return F.log_softmax(x.float(), dim=dim) if onnx_trace else F.log_softmax(x, dim=dim, dtype=torch.float32) - -def eval_str_dict(x, type=dict): - if x is None: return None - if isinstance(x, str): x = eval(x) - - return x - -def with_incremental_state(cls): - cls.__bases__ = (FairseqIncrementalState,) + tuple(b for b in cls.__bases__ if b != FairseqIncrementalState) - return cls - -def quant_noise(module, p, block_size): - if p <= 0: return module - assert isinstance(module, (nn.Linear, nn.Embedding, nn.Conv2d)) - is_conv = module.weight.ndim == 4 - - if not is_conv: assert (module.weight.size(1) % block_size == 0) - else: - if module.kernel_size == (1, 1): assert (module.in_channels % block_size == 0) - else: - k = module.kernel_size[0] * module.kernel_size[1] - assert k % block_size == 0 - - def _forward_pre_hook(mod, input): - if mod.training: - if not is_conv: - weight = mod.weight - - in_features = weight.size(1) - out_features = weight.size(0) - - mask = torch.zeros(in_features // block_size * out_features, device=weight.device) - mask.bernoulli_(p) - mask = mask.repeat_interleave(block_size, -1).view(-1, in_features) - else: - weight = mod.weight - - in_channels = mod.in_channels - out_channels = mod.out_channels - - if mod.kernel_size == (1, 1): - mask = torch.zeros(int(in_channels // block_size * out_channels), device=weight.device) - mask.bernoulli_(p) - mask = mask.repeat_interleave(block_size, -1).view(-1, in_channels) - else: - mask = torch.zeros(weight.size(0), weight.size(1), device=weight.device) - mask.bernoulli_(p) - mask = (mask.unsqueeze(2).unsqueeze(3).repeat(1, 1, mod.kernel_size[0], mod.kernel_size[1])) - - mask = mask.to(torch.bool) - s = 1 / (1 - p) - mod.weight.data = s * weight.masked_fill(mask, 0) - - module.register_forward_pre_hook(_forward_pre_hook) - return module - -class FairseqDropout(nn.Module): - def __init__(self, p, module_name=None): - super().__init__() - self.p = p - self.module_name = module_name - self.apply_during_inference = False - - def forward(self, x, inplace = False): - return F.dropout(x, p=self.p, training=True, inplace=inplace) if self.p > 0 and (self.training or self.apply_during_inference) else x - - def make_generation_fast_(self, name, retain_dropout = False, retain_dropout_modules = None, **kwargs): - if retain_dropout and (retain_dropout_modules is None or self.module_name in retain_dropout_modules): self.apply_during_inference = True - -class FairseqIncrementalState(object): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.init_incremental_state() - - def init_incremental_state(self): - self._incremental_state_id = str(uuid.uuid4()) - - def _get_full_incremental_state_key(self, key): - return "{}.{}".format(self._incremental_state_id, key) - - def get_incremental_state(self, incremental_state, key): - full_key = self._get_full_incremental_state_key(key) - if incremental_state is None or full_key not in incremental_state: return None - return incremental_state[full_key] - - def set_incremental_state(self, incremental_state, key, value): - if incremental_state is not None: incremental_state[self._get_full_incremental_state_key(key)] = value - return incremental_state - -class FairseqDecoder(nn.Module): - def __init__(self, dictionary): - super().__init__() - self.dictionary = dictionary - self.onnx_trace = False - self.adaptive_softmax = None - - def forward(self, prev_output_tokens, encoder_out=None, **kwargs): - x, extra = self.extract_features(prev_output_tokens, encoder_out=encoder_out, **kwargs) - return self.output_layer(x), extra - - def extract_features(self, prev_output_tokens, encoder_out=None, **kwargs): - pass - - def output_layer(self, features, **kwargs): - pass - - def get_normalized_probs(self, net_output, log_probs, sample = None): - return self.get_normalized_probs_scriptable(net_output, log_probs, sample) - - def get_normalized_probs_scriptable(self, net_output, log_probs, sample = None): - if hasattr(self, "adaptive_softmax") and self.adaptive_softmax is not None: - if sample is not None: - assert "target" in sample - target = sample["target"] - else: target = None - - out = self.adaptive_softmax.get_log_prob(net_output[0], target=target) - return out.exp_() if not log_probs else out - - logits = net_output[0] - return log_softmax(logits, dim=-1, onnx_trace=self.onnx_trace) if log_probs else softmax(logits, dim=-1, onnx_trace=self.onnx_trace) - - def max_positions(self): - return 1e6 - - def upgrade_state_dict_named(self, state_dict, name): - return state_dict - - def prepare_for_onnx_export_(self): - self.onnx_trace = True - -@with_incremental_state -class FairseqIncrementalDecoder(FairseqDecoder): - def __init__(self, dictionary): - super().__init__(dictionary) - - def forward(self, prev_output_tokens, encoder_out=None, incremental_state=None, **kwargs): - pass - - def extract_features(self, prev_output_tokens, encoder_out=None, incremental_state=None, **kwargs): - pass - - def reorder_incremental_state(self, incremental_state, new_order): - pass - - def reorder_incremental_state_scripting(self, incremental_state, new_order): - for module in self.modules(): - if hasattr(module, "reorder_incremental_state"): - result = module.reorder_incremental_state(incremental_state, new_order) - if result is not None: incremental_state = result - - def set_beam_size(self, beam_size): - if getattr(self, "_beam_size", -1) != beam_size: - seen = set() - - def apply_set_beam_size(module): - if (module != self and hasattr(module, "set_beam_size") and module not in seen): - seen.add(module) - module.set_beam_size(beam_size) - - self.apply(apply_set_beam_size) - self._beam_size = beam_size - -class MultiheadAttention(FairseqIncrementalDecoder): - def __init__( - self, - embed_dim, - num_heads, - kdim=None, - vdim=None, - dropout=0.0, - bias=True, - add_bias_kv=False, - add_zero_attn=False, - self_attention=False, - encoder_decoder_attention=False, - dictionary=None, - q_noise=0.0, - qn_block_size=8, - xformers_att_config=None, - xformers_blocksparse_layout=None, - xformers_blocksparse_blocksize=16 - ): - super().__init__(dictionary) - xformers_att_config = eval_str_dict(xformers_att_config) - self.use_xformers = xformers_att_config is not None - if self.use_xformers: raise ImportError - self.embed_dim = embed_dim - self.kdim = kdim if kdim is not None else embed_dim - self.vdim = vdim if vdim is not None else embed_dim - self.qkv_same_dim = self.kdim == embed_dim and self.vdim == embed_dim - self.num_heads = num_heads - self.dropout_module = FairseqDropout(dropout, module_name=self.__class__.__name__) - self.head_dim = embed_dim // num_heads - assert (self.head_dim * num_heads == self.embed_dim) - self.scaling = self.head_dim**-0.5 - self.self_attention = self_attention - self.encoder_decoder_attention = encoder_decoder_attention - assert not self.self_attention or self.qkv_same_dim - self.k_proj = quant_noise(nn.Linear(self.kdim, embed_dim, bias=bias), q_noise, qn_block_size) - self.v_proj = quant_noise(nn.Linear(self.vdim, embed_dim, bias=bias), q_noise, qn_block_size) - self.q_proj = quant_noise(nn.Linear(embed_dim, embed_dim, bias=bias), q_noise, qn_block_size) - self.out_proj = quant_noise(nn.Linear(embed_dim, embed_dim, bias=bias), q_noise, qn_block_size) - if add_bias_kv: self.bias_k, self.bias_v = nn.Parameter(torch.Tensor(1, 1, embed_dim)), nn.Parameter(torch.Tensor(1, 1, embed_dim)) - else: self.bias_k = self.bias_v = None - self.add_zero_attn = add_zero_attn - self.beam_size = 1 - self.reset_parameters() - self.onnx_trace = False - self.skip_embed_dim_check = False - self.init_incremental_state() - - def prepare_for_onnx_export_(self): - self.onnx_trace = True - - def reset_parameters(self): - if self.qkv_same_dim: - nn.init.xavier_uniform_(self.k_proj.weight, gain=1 / math.sqrt(2)) - nn.init.xavier_uniform_(self.v_proj.weight, gain=1 / math.sqrt(2)) - nn.init.xavier_uniform_(self.q_proj.weight, gain=1 / math.sqrt(2)) - else: - nn.init.xavier_uniform_(self.k_proj.weight) - nn.init.xavier_uniform_(self.v_proj.weight) - nn.init.xavier_uniform_(self.q_proj.weight) - - nn.init.xavier_uniform_(self.out_proj.weight) - if self.out_proj.bias is not None: nn.init.constant_(self.out_proj.bias, 0.0) - if self.bias_k is not None: nn.init.xavier_normal_(self.bias_k) - if self.bias_v is not None: nn.init.xavier_normal_(self.bias_v) - - def _get_reserve_head_index(self, num_heads_to_keep: int): - k_proj_heads_norm, q_proj_heads_norm, v_proj_heads_norm = [], [], [] - - for i in range(self.num_heads): - start_idx = i * self.head_dim - end_idx = (i + 1) * self.head_dim - - k_proj_heads_norm.append( - (self.k_proj.weight[start_idx:end_idx]).abs().sum().tolist() + - (self.k_proj.bias[start_idx:end_idx]).abs().sum().tolist() - ) - q_proj_heads_norm.append( - (self.q_proj.weight[start_idx:end_idx]).abs().sum().tolist() + - (self.q_proj.bias[start_idx:end_idx]).abs().sum().tolist() - ) - v_proj_heads_norm.append( - (self.v_proj.weight[start_idx:end_idx]).abs().sum().tolist() + - (self.v_proj.bias[start_idx:end_idx]).abs().sum().tolist() - ) - - heads_norm = [] - for i in range(self.num_heads): - heads_norm.append(k_proj_heads_norm[i] + q_proj_heads_norm[i] + v_proj_heads_norm[i]) - - sorted_head_index = sorted(range(self.num_heads), key=lambda k: heads_norm[k], reverse=True) - reserve_head_index = [] - for i in range(num_heads_to_keep): - reserve_head_index.append((sorted_head_index[i] * self.head_dim, (sorted_head_index[i] + 1) * self.head_dim)) - - return reserve_head_index - - def _adaptive_prune_heads(self, reserve_head_index): - new_q_weight, new_q_bias, new_k_weight, new_k_bias, new_v_weight, new_v_bias, new_out_proj_weight = [], [], [], [], [], [], [] - - for ele in reserve_head_index: - start_idx, end_idx = ele - - new_q_weight.append(self.q_proj.weight[start_idx:end_idx]) - new_q_bias.append(self.q_proj.bias[start_idx:end_idx]) - - new_k_weight.append(self.k_proj.weight[start_idx:end_idx]) - new_k_bias.append(self.k_proj.bias[start_idx:end_idx]) - - new_v_weight.append(self.v_proj.weight[start_idx:end_idx]) - new_v_bias.append(self.v_proj.bias[start_idx:end_idx]) - - new_out_proj_weight.append(self.out_proj.weight[:, start_idx:end_idx]) - - new_q_weight = torch.cat(new_q_weight).detach() - new_k_weight = torch.cat(new_k_weight).detach() - new_v_weight = torch.cat(new_v_weight).detach() - - new_out_proj_weight = torch.cat(new_out_proj_weight, dim=-1).detach() - - new_q_weight.requires_grad = True - new_k_weight.requires_grad = True - new_v_weight.requires_grad = True - - new_out_proj_weight.requires_grad = True - - new_q_bias = torch.cat(new_q_bias).detach() - new_q_bias.requires_grad = True - - new_k_bias = torch.cat(new_k_bias).detach() - new_k_bias.requires_grad = True - - new_v_bias = torch.cat(new_v_bias).detach() - new_v_bias.requires_grad = True - - self.q_proj.weight = nn.Parameter(new_q_weight) - self.q_proj.bias = nn.Parameter(new_q_bias) - - self.k_proj.weight = nn.Parameter(new_k_weight) - self.k_proj.bias = nn.Parameter(new_k_bias) - - self.v_proj.weight = nn.Parameter(new_v_weight) - self.v_proj.bias = nn.Parameter(new_v_bias) - - self.out_proj.weight = nn.Parameter(new_out_proj_weight) - self.num_heads = len(reserve_head_index) - self.embed_dim = self.head_dim * self.num_heads - - self.q_proj.out_features = self.embed_dim - self.k_proj.out_features = self.embed_dim - self.v_proj.out_features = self.embed_dim - - def _set_skip_embed_dim_check(self): - self.skip_embed_dim_check = True - - def _pad_masks(self, key_padding_mask, attn_mask): - if attn_mask is not None: - shape = attn_mask.size()[:-1] + torch.Size([1]) - attn_mask = torch.cat([attn_mask, attn_mask.new_zeros(shape)], dim=-1) - - if key_padding_mask is not None: - shape = key_padding_mask.size()[:-1] + torch.Size([1]) - key_padding_mask = torch.cat([key_padding_mask, key_padding_mask.new_zeros(shape)], dim=-1) - - return key_padding_mask, attn_mask - - def _add_bias(self, k, v, key_padding_mask, attn_mask, bsz): - assert self.bias_k is not None or self.bias_v is not None - key_padding_mask, attn_mask = self._pad_masks(key_padding_mask=key_padding_mask, attn_mask=attn_mask) - - return torch.cat([k, self.bias_k.repeat(1, bsz, 1)]), torch.cat([v, self.bias_v.repeat(1, bsz, 1)]), key_padding_mask, attn_mask - - def _append_zero_attn(self, k, v, key_padding_mask, attn_mask): - zero_attn_shape = k.size()[:-2] + torch.Size([1]) + k.size()[-1:] - key_padding_mask, attn_mask = self._pad_masks(key_padding_mask=key_padding_mask, attn_mask=attn_mask) - - return torch.cat([k, torch.zeros(zero_attn_shape, dtype=k.dtype, device=k.device)], dim=-2), torch.cat([v, torch.zeros(zero_attn_shape, dtype=v.dtype, device=v.device)], dim=-2), key_padding_mask, attn_mask - - def forward(self, query, key, value, key_padding_mask = None, incremental_state = None, need_weights = True, static_kv = False, attn_mask = None, before_softmax = False, need_head_weights = False): - if need_head_weights: need_weights = True - - tgt_len, bsz, embed_dim = query.size() - src_len = tgt_len - - if not self.skip_embed_dim_check: assert (embed_dim == self.embed_dim) - assert list(query.size()) == [tgt_len, bsz, embed_dim] - - if key is not None: - src_len, key_bsz, _ = key.size() - if not torch.jit.is_scripting(): - assert value is not None - assert src_len, key_bsz == value.shape[:2] - - if (not self.onnx_trace and incremental_state is None and not static_kv and not torch.jit.is_scripting() and not self.skip_embed_dim_check): - assert key is not None and value is not None - - return F.multi_head_attention_forward( - query, - key, - value, - self.embed_dim, - self.num_heads, - torch.empty([0]), - torch.cat((self.q_proj.bias, self.k_proj.bias, self.v_proj.bias)), - self.bias_k, - self.bias_v, - self.add_zero_attn, - self.dropout_module.p, - self.out_proj.weight, - self.out_proj.bias, - self.training or self.dropout_module.apply_during_inference, - key_padding_mask.bool() if key_padding_mask is not None else None, - need_weights, - attn_mask, - use_separate_proj_weight=True, - q_proj_weight=self.q_proj.weight, - k_proj_weight=self.k_proj.weight, - v_proj_weight=self.v_proj.weight - ) - - if incremental_state is not None: - saved_state = self._get_input_buffer(incremental_state) - - if saved_state is not None and "prev_key" in saved_state: - if static_kv: - assert self.encoder_decoder_attention and not self.self_attention - key = value = None - else: saved_state = None - - if self.self_attention: - q = self.q_proj(query) - k = self.k_proj(query) - v = self.v_proj(query) - elif self.encoder_decoder_attention: - q = self.q_proj(query) - if key is None: - assert value is None - k = v = None - else: - if self.beam_size > 1 and bsz == key.size(1): - key = key.view(key.size(0), -1, self.beam_size, key.size(2))[:, :, 0, :] - if key_padding_mask is not None: key_padding_mask = key_padding_mask.view(-1, self.beam_size, key_padding_mask.size(1))[:, 0, :] - - k = self.k_proj(key) - v = self.v_proj(key) - else: - assert key is not None and value is not None - q = self.q_proj(query) - k = self.k_proj(key) - v = self.v_proj(value) - - q *= self.scaling - if self.bias_k is not None: - assert self.bias_v is not None - k, v, attn_mask, key_padding_mask = self._add_bias(k, v, attn_mask, key_padding_mask, bsz) - - q = (q.contiguous().view(tgt_len, bsz * self.num_heads, self.head_dim).transpose(0, 1)) - kv_bsz = bsz - - if k is not None: - kv_bsz = k.size(1) - k = (k.contiguous().view(-1, kv_bsz * self.num_heads, self.head_dim).transpose(0, 1)) - - if v is not None: v = (v.contiguous().view(-1, kv_bsz * self.num_heads, self.head_dim).transpose(0, 1)) - if saved_state is not None: - if "prev_key" in saved_state: - _prev_key = saved_state["prev_key"] - assert _prev_key is not None - - kv_bsz = _prev_key.size(0) - prev_key = _prev_key.view(kv_bsz * self.num_heads, -1, self.head_dim) - - if static_kv: k = prev_key - else: - assert k is not None - k = torch.cat([prev_key, k], dim=1) - - src_len = k.size(1) - - if "prev_value" in saved_state: - _prev_value = saved_state["prev_value"] - assert _prev_value is not None or kv_bsz == _prev_value.size(0) - prev_value = _prev_value.view(kv_bsz * self.num_heads, -1, self.head_dim) - if static_kv: v = prev_value - else: - assert v is not None - v = torch.cat([prev_value, v], dim=1) - - prev_key_padding_mask = None - if "prev_key_padding_mask" in saved_state: prev_key_padding_mask = saved_state["prev_key_padding_mask"] - assert k is not None and v is not None - - key_padding_mask = MultiheadAttention._append_prev_key_padding_mask( - key_padding_mask=key_padding_mask, - prev_key_padding_mask=prev_key_padding_mask, - batch_size=kv_bsz, - src_len=k.size(1), - static_kv=static_kv - ) - - saved_state["prev_key"] = k.view(kv_bsz, self.num_heads, -1, self.head_dim) - saved_state["prev_value"] = v.view(kv_bsz, self.num_heads, -1, self.head_dim) - saved_state["prev_key_padding_mask"] = key_padding_mask - - assert incremental_state is not None - incremental_state = self._set_input_buffer(incremental_state, saved_state) - - assert k is not None - assert k.size(1) == src_len - - if key_padding_mask is not None and key_padding_mask.dim() == 0: key_padding_mask = None - - if key_padding_mask is not None: - assert key_padding_mask.size(0) == kv_bsz - assert key_padding_mask.size(1) == src_len - - if self.add_zero_attn: - assert v is not None - src_len += 1 - k, v, key_padding_mask, attn_mask = self._append_zero_attn(k=k, v=v, key_padding_mask=key_padding_mask, attn_mask=attn_mask) - - if self.encoder_decoder_attention and bsz != kv_bsz: - attn_weights = torch.einsum( - "bxhtd,bhsd->bxhts", - q.view((kv_bsz, -1, self.num_heads) + q.size()[1:]), - k.view((kv_bsz, self.num_heads) + k.size()[1:]) - ) - - attn_weights = attn_weights.reshape((-1,) + attn_weights.size()[-2:]) - else: attn_weights = q.bmm(k.transpose(1, 2)) - - assert list(attn_weights.size()) == [bsz * self.num_heads, tgt_len, src_len] - - if attn_mask is not None: - attn_mask = attn_mask.unsqueeze(0) - if self.onnx_trace: attn_mask = attn_mask.repeat(attn_weights.size(0), 1, 1) - attn_weights += attn_mask - - if key_padding_mask is not None: - attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) - attn_weights = attn_weights.view(kv_bsz, -1, self.num_heads, tgt_len, src_len) - attn_weights = attn_weights.masked_fill(key_padding_mask.unsqueeze(1).unsqueeze(2).unsqueeze(3).to(torch.bool), float("-inf")) - attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) - - if before_softmax: return attn_weights, v - - attn_weights_float = softmax(attn_weights, dim=-1, onnx_trace=self.onnx_trace) - attn_weights = attn_weights_float.type_as(attn_weights) - attn_probs = self.dropout_module(attn_weights) - - assert v is not None - attn = None - - if self.encoder_decoder_attention and bsz != kv_bsz: - attn = torch.einsum( - "bxhts,bhsd->bxhtd", - attn_probs.view((kv_bsz, -1, self.num_heads) + attn_probs.size()[1:]), - v.view((kv_bsz, self.num_heads) + v.size()[1:]) - ) - - attn = attn.reshape((-1,) + attn.size()[-2:]) - else: attn = attn_probs.bmm(v) - assert list(attn.size()) == [bsz * self.num_heads, tgt_len, self.head_dim] - - attn = attn.contiguous().view(tgt_len, bsz, self.embed_dim) if self.onnx_trace and attn.size(1) == 1 else attn.transpose(0, 1).contiguous().view(tgt_len, bsz, self.embed_dim) - attn = self.out_proj(attn) - attn_weights = None - - if need_weights: - attn_weights = attn_weights_float.view(bsz, self.num_heads, tgt_len, src_len).transpose(1, 0) - if not need_head_weights: attn_weights = attn_weights.mean(dim=0) - - return attn, attn_weights - - @staticmethod - def _append_prev_key_padding_mask(key_padding_mask, prev_key_padding_mask, batch_size, src_len, static_kv): - if prev_key_padding_mask is not None and static_kv: - new_key_padding_mask = prev_key_padding_mask - elif prev_key_padding_mask is not None and key_padding_mask is not None: - new_key_padding_mask = torch.cat([prev_key_padding_mask.float(), key_padding_mask.float()], dim=1) - elif prev_key_padding_mask is not None: - if src_len > prev_key_padding_mask.size(1): - filler = torch.zeros((batch_size, src_len - prev_key_padding_mask.size(1)), device=prev_key_padding_mask.device) - new_key_padding_mask = torch.cat([prev_key_padding_mask.float(), filler.float()], dim=1) - else: - new_key_padding_mask = prev_key_padding_mask.float() - elif key_padding_mask is not None: - if src_len > key_padding_mask.size(1): - filler = torch.zeros((batch_size, src_len - key_padding_mask.size(1)), device=key_padding_mask.device) - new_key_padding_mask = torch.cat([filler.float(), key_padding_mask.float()], dim=1) - else: - new_key_padding_mask = key_padding_mask.float() - else: - new_key_padding_mask = prev_key_padding_mask - - return new_key_padding_mask - - @torch.jit.export - def reorder_incremental_state(self, incremental_state, new_order): - input_buffer = self._get_input_buffer(incremental_state) - - if input_buffer is not None: - for k in input_buffer.keys(): - input_buffer_k = input_buffer[k] - - if input_buffer_k is not None: - if self.encoder_decoder_attention: - if input_buffer_k.size(0) * self.beam_size == new_order.size(0): - return incremental_state - elif self.beam_size > 1: - input_buffer[k] = input_buffer_k.index_select(0, new_order.reshape(-1, self.beam_size)[:, 0] // self.beam_size) - else: - input_buffer[k] = input_buffer_k.index_select(0, new_order) - else: - input_buffer[k] = input_buffer_k.index_select(0, new_order) - - incremental_state = self._set_input_buffer(incremental_state, input_buffer) - - return incremental_state - - def set_beam_size(self, beam_size): - self.beam_size = beam_size - - def _get_input_buffer(self, incremental_state): - result = self.get_incremental_state(incremental_state, "attn_state") - return result if result is not None else {} - - def _set_input_buffer(self, incremental_state, buffer): - return self.set_incremental_state(incremental_state, "attn_state", buffer) - - def upgrade_state_dict_named(self, state_dict, name): - prefix = name + "." if name != "" else "" - items_to_add, keys_to_remove = {}, [] - for k in state_dict.keys(): - if k.endswith(prefix + "in_proj_weight"): - dim = int(state_dict[k].shape[0] / 3) - - items_to_add[prefix + "q_proj.weight"] = state_dict[k][:dim] - items_to_add[prefix + "k_proj.weight"] = state_dict[k][dim : 2 * dim] - items_to_add[prefix + "v_proj.weight"] = state_dict[k][2 * dim :] - - keys_to_remove.append(k) - k_bias = prefix + "in_proj_bias" - - if k_bias in state_dict.keys(): - dim = int(state_dict[k].shape[0] / 3) - items_to_add[prefix + "q_proj.bias"] = state_dict[k_bias][:dim] - items_to_add[prefix + "k_proj.bias"] = state_dict[k_bias][dim : 2 * dim] - items_to_add[prefix + "v_proj.bias"] = state_dict[k_bias][2 * dim :] - keys_to_remove.append(prefix + "in_proj_bias") - - for k in keys_to_remove: - del state_dict[k] - - for key, value in items_to_add.items(): - state_dict[key] = value - -def init_bert_params(module): - def normal_(data): - data.copy_(data.cpu().normal_(mean=0.0, std=0.02).to(data.device)) - - if isinstance(module, nn.Linear): - normal_(module.weight.data) - if module.bias is not None: module.bias.data.zero_() - - if isinstance(module, nn.Embedding): - normal_(module.weight.data) - if module.padding_idx is not None: module.weight.data[module.padding_idx].zero_() - - if isinstance(module, MultiheadAttention): - normal_(module.q_proj.weight.data) - normal_(module.k_proj.weight.data) - normal_(module.v_proj.weight.data) - -def make_conv_pos(e, k, g): - pos_conv = nn.Conv1d(e, e, kernel_size=k, padding=k // 2, groups=g) - dropout = 0 - - nn.init.normal_(pos_conv.weight, mean=0, std=math.sqrt((4 * (1.0 - dropout)) / (k * e))) - nn.init.constant_(pos_conv.bias, 0) - - return nn.Sequential(nn.utils.parametrizations.weight_norm(pos_conv, name="weight", dim=2), SamePad(k), nn.GELU()) - -def index_put(tensor, indices, value): - tensor[indices] = value - return tensor - -def pad_to_multiple(x, multiple, dim=-1, value=0): - if x is None: return None, 0 - tsz = x.size(dim) - m = tsz / multiple - remainder = math.ceil(m) * multiple - tsz - if m.is_integer(): return x, 0 - return F.pad(x, (*((0,) * (-1 - dim) * 2), 0, remainder), value=value), remainder - -def compute_mask_indices( - shape, - padding_mask, - mask_prob, - mask_length, - mask_type = "static", - mask_other = 0.0, - min_masks = 0, - no_overlap = False, - min_space = 0, - require_same_masks = True, - mask_dropout = 0.0, - add_masks = False, - seed = None, - epoch = None, - indices = None, - idc_select_ver = 1, - num_mask_ver = 2 -): - bsz, all_sz = shape - mask = np.full((bsz, all_sz), False) - if num_mask_ver == 1: all_num_mask = max(min_masks, int(mask_prob * all_sz / float(mask_length) + np.random.rand())) - mask_idcs = [] - - for i in range(bsz): - seed_i = int(hash((seed, epoch, indices[i].item())) % 1e6) if seed is not None and epoch is not None and indices is not None else None - rng = np.random.default_rng(seed_i) - - if padding_mask is not None: - sz = all_sz - padding_mask[i].long().sum().item() - assert sz >= 0, sz - else: sz = all_sz - - if num_mask_ver == 1: num_mask = max(min_masks, int(mask_prob * sz / float(mask_length) + np.random.rand())) if padding_mask is not None else all_num_mask - elif num_mask_ver == 2: num_mask = max(min_masks, int(mask_prob * sz / float(mask_length) + rng.random())) - else: raise ValueError - - if mask_type == "static": lengths = np.full(num_mask, mask_length) - elif mask_type == "uniform": lengths = rng.randint(mask_other, mask_length * 2 + 1, size=num_mask) - elif mask_type == "normal": lengths = [max(1, int(round(x))) for x in rng.normal(mask_length, mask_other, size=num_mask)] - elif mask_type == "poisson": lengths = [int(round(x)) for x in rng.poisson(mask_length, size=num_mask)] - else: raise Exception - - if sum(lengths) == 0: - if mask_type == "static": raise ValueError - else: lengths = [min(mask_length, sz - 1)] - - if no_overlap: - mask_idc = [] - - def arrange(s, e, length, keep_length): - span_start = rng.randint(s, e - length) - mask_idc.extend(span_start + i for i in range(length)) - new_parts = [] - if span_start - s - min_space >= keep_length: new_parts.append((s, span_start - min_space + 1)) - if e - span_start - length - min_space > keep_length: new_parts.append((span_start + length + min_space, e)) - return new_parts - - parts = [(0, sz)] - min_length = min(lengths) - for length in sorted(lengths, reverse=True): - lens = np.fromiter((e - s if e - s >= length + min_space else 0 for s, e in parts), np.int32) - l_sum = np.sum(lens) - if l_sum == 0: break - s, e = parts.pop(rng.choice(len(parts), p=lens / np.sum(lens))) - parts.extend(arrange(s, e, length, min_length)) - mask_idc = np.asarray(mask_idc) - else: - if idc_select_ver == 1: - min_len = min(lengths) - if sz - min_len <= num_mask: min_len = sz - num_mask - 1 - mask_idc = rng.choice(sz - min_len, num_mask, replace=False) - elif idc_select_ver == 2: mask_idc = rng.choice(sz, num_mask, replace=False) - else: raise ValueError - - mask_idc = np.asarray([mask_idc[j] + offset for j in range(len(mask_idc)) for offset in range(lengths[j])]) - - mask_idc = np.unique(mask_idc[mask_idc < sz]) - if len(mask_idc) >= sz: raise ValueError - mask_idcs.append(mask_idc) - - target_len = None - if require_same_masks: target_len = max([len(m) for m in mask_idcs]) if add_masks else min([len(m) for m in mask_idcs]) - - for i, mask_idc in enumerate(mask_idcs): - if target_len is not None and len(mask_idc) > target_len: mask_idc = rng.choice(mask_idc, target_len, replace=False) - mask[i, mask_idc] = True - - if target_len is not None and len(mask_idc) < target_len: - to_mask = rng.choice(np.flatnonzero(~mask[i]), target_len - len(mask_idc), replace=False) - mask[i, to_mask] = True - - if mask_dropout > 0: - masked = np.flatnonzero(mask[i]) - mask[i, rng.choice(masked, np.rint(len(masked) * mask_dropout).astype(int), replace=False)] = False - - return mask - -def LayerNorm(normalized_shape, eps=1e-5, elementwise_affine=True): - return nn.LayerNorm(normalized_shape, eps, elementwise_affine) - -def prune_state_dict(state_dict, model_cfg): - arch = None - if model_cfg is not None: arch = (model_cfg._name if isinstance(model_cfg, DictConfig) else getattr(model_cfg, "arch", None)) - if not model_cfg or arch is None or arch == "ptt_transformer": return state_dict - encoder_layers_to_keep = getattr(model_cfg, "encoder_layers_to_keep", None) - decoder_layers_to_keep = getattr(model_cfg, "decoder_layers_to_keep", None) - if not encoder_layers_to_keep and not decoder_layers_to_keep: return state_dict - - def create_pruning_pass(layers_to_keep, layer_name): - keep_layers = sorted(int(layer_string) for layer_string in layers_to_keep.split(",")) - mapping_dict = {} - for i in range(len(keep_layers)): - mapping_dict[str(keep_layers[i])] = str(i) - - return {"substitution_regex": re.compile(r"^{layer}.*\.layers\.(\d+)".format(layer=layer_name)), "mapping_dict": mapping_dict} - - pruning_passes, new_state_dict = [], {} - if encoder_layers_to_keep: pruning_passes.append(create_pruning_pass(encoder_layers_to_keep, "encoder")) - if decoder_layers_to_keep: pruning_passes.append(create_pruning_pass(decoder_layers_to_keep, "decoder")) - - for layer_name in state_dict.keys(): - match = re.search(r"\.layers\.(\d+)\.", layer_name) - if not match: - new_state_dict[layer_name] = state_dict[layer_name] - continue - - original_layer_number = match.group(1) - for pruning_pass in pruning_passes: - if original_layer_number in pruning_pass["mapping_dict"] and pruning_pass["substitution_regex"].search(layer_name): - substitution_match = pruning_pass["substitution_regex"].search(layer_name) - new_state_dict[ - ( - layer_name[: substitution_match.start(1)] + - pruning_pass["mapping_dict"][original_layer_number] + - layer_name[substitution_match.end(1) :] - ) - ] = state_dict[layer_name] - - with open_dict(model_cfg) if isinstance(model_cfg, DictConfig) else contextlib.ExitStack(): - if hasattr(model_cfg, "encoder_layers_to_keep"): model_cfg.encoder_layers_to_keep = None - if hasattr(model_cfg, "decoder_layers_to_keep"): model_cfg.decoder_layers_to_keep = None - - return new_state_dict - -def relu_squared(x): - return F.relu(x).pow(2) - -def get_activation_fn(activation): - def gelu(x): - return nn.functional.gelu(x.float()).type_as(x) - - def gelu_accurate(x): - if not hasattr(gelu_accurate, "_a"): - gelu_accurate._a = math.sqrt(2 / math.pi) - return (0.5 * x * (1 + (gelu_accurate._a * (x + 0.044715 * x.pow(3))).tanh())) - - if activation == "relu": return F.relu - elif activation == "relu_squared": return relu_squared - elif activation == "gelu": return gelu - elif activation == "gelu_fast": return gelu_accurate - elif activation == "gelu_accurate": return gelu_accurate - elif activation == "tanh": return torch.tanh - elif activation == "linear": return lambda x: x - elif activation == "swish": return nn.SiLU - else: raise RuntimeError - -class SamePad(nn.Module): - def __init__( - self, - kernel_size, - causal=False - ): - super().__init__() - if causal: self.remove = kernel_size - 1 - else: self.remove = int(kernel_size % 2 == 0) - - def forward(self, x): - if self.remove > 0: x = x[:, :, : -self.remove] - return x - -class TransformerSentenceEncoderLayer(nn.Module): - def __init__( - self, - embedding_dim = 768, - ffn_embedding_dim = 3072, - num_attention_heads = 8, - dropout = 0.1, - attention_dropout = 0.1, - activation_dropout = 0.1, - activation_fn = "relu", - layer_norm_first = False - ): - super().__init__() - self.embedding_dim = embedding_dim - self.dropout = dropout - self.activation_dropout = activation_dropout - self.activation_fn = get_activation_fn(activation_fn) - self.self_attn = MultiheadAttention(self.embedding_dim, num_attention_heads, dropout=attention_dropout, self_attention=True) - self.dropout1 = nn.Dropout(dropout) - self.dropout2 = nn.Dropout(self.activation_dropout) - self.dropout3 = nn.Dropout(dropout) - self.layer_norm_first = layer_norm_first - self.self_attn_layer_norm = LayerNorm(self.embedding_dim) - self.fc1 = nn.Linear(self.embedding_dim, ffn_embedding_dim) - self.fc2 = nn.Linear(ffn_embedding_dim, self.embedding_dim) - self.final_layer_norm = LayerNorm(self.embedding_dim) - - def forward(self, x, self_attn_mask=None, self_attn_padding_mask=None, need_weights=False, att_args=None): - residual = x - if self.layer_norm_first: - x = self.self_attn_layer_norm(x) - x, attn = self.self_attn(query=x, key=x, value=x, key_padding_mask=self_attn_padding_mask, attn_mask=self_attn_mask, need_weights=False) - x = residual + self.dropout1(x) - - residual = x - x = self.fc2(self.dropout2(self.activation_fn(self.fc1(self.final_layer_norm(x))))) - - layer_result = x - x = residual + self.dropout3(x) - else: - x, attn = self.self_attn(query=x, key=x, value=x, key_padding_mask=self_attn_padding_mask, need_weights=False) - x = self.self_attn_layer_norm(residual + self.dropout1(x)) - - residual = x - x = self.fc2(self.dropout2(self.activation_fn(self.fc1(x)))) - - layer_result = x - x = self.final_layer_norm(residual + self.dropout3(x)) - - return x, (attn, layer_result) - -class AdapterFast(nn.Module): - def __init__( - self, - adapter_num, - input_dim, - hidden_dim, - act_fn - ): - super().__init__() - self.adapter_num = adapter_num - self.input_dim = input_dim - self.hidden_dim = hidden_dim - self.W_a = nn.Parameter(torch.empty(adapter_num, hidden_dim, input_dim)) - self.W_b = nn.Parameter(torch.empty(adapter_num, input_dim, hidden_dim)) - self.b_a = nn.Parameter(torch.empty(adapter_num, hidden_dim)) - self.b_b = nn.Parameter(torch.empty(adapter_num, input_dim)) - self.ln_W = nn.Parameter(torch.empty(adapter_num, input_dim)) - self.ln_b = nn.Parameter(torch.empty(adapter_num, input_dim)) - self.act_fn = nn.Identity() - if act_fn == "relu": self.act_fn = nn.ReLU() - elif act_fn == "gelu": self.act_fn = nn.GELU() - elif act_fn == "selu": self.act_fn = nn.SELU() - else: raise ValueError - self.input_dim = input_dim - self.reset_parameters() - - def reset_parameters(self): - for ii in range(self.adapter_num): - nn.init.kaiming_uniform_(self.W_a[ii], a=math.sqrt(5)) - nn.init.kaiming_uniform_(self.W_b[ii], a=math.sqrt(5)) - fan_in, _ = nn.init._calculate_fan_in_and_fan_out(self.W_a[ii]) - bound = 1 / math.sqrt(fan_in) if fan_in > 0 else 0 - nn.init.uniform_(self.b_a[ii], -bound, bound) - fan_in, _ = nn.init._calculate_fan_in_and_fan_out(self.W_b[ii]) - bound = 1 / math.sqrt(fan_in) if fan_in > 0 else 0 - nn.init.uniform_(self.b_b[ii], -bound, bound) - - nn.init.ones_(self.ln_W) - nn.init.zeros_(self.ln_b) - - def forward(self, x, adapter_id): - ii = adapter_id - return F.linear( - self.act_fn( - F.linear( - F.layer_norm( - x, - (self.input_dim, ), - self.ln_W[ii], - self.ln_b[ii] - ), - self.W_a[ii], - self.b_a[ii] - ) - ), - self.W_b[ii], - self.b_b[ii] - ) - - def extra_repr(self): - return ( - 'adapter={}, input_dim={}, hidden_dim={}'.format(self.adapter_num, self.input_dim, self.hidden_dim) - ) - -class FeedForwardModule(nn.Module): - def __init__( - self, - input_feat, - hidden_units, - dropout1, - dropout2, - activation_fn="swish", - bias=True - ): - super(FeedForwardModule, self).__init__() - self.layer_norm = LayerNorm(input_feat) - self.w_1 = nn.Linear(input_feat, hidden_units, bias=bias) - self.w_2 = nn.Linear(hidden_units, input_feat, bias=bias) - self.dropout1 = nn.Dropout(dropout1) - self.dropout2 = nn.Dropout(dropout2) - self.activation = get_activation_fn(activation_fn)(hidden_units) - - def forward(self, x): - return self.dropout2( - self.w_2( - self.dropout1( - self.activation( - self.w_1( - self.layer_norm(x) - ) - ) - ) - ) - ) - -class ConvolutionModule(nn.Module): - def __init__( - self, - embed_dim, - channels, - depthwise_kernel_size, - dropout, - activation_fn="swish", - bias=False, - export=False - ): - super(ConvolutionModule, self).__init__() - assert (depthwise_kernel_size - 1) % 2 == 0 - self.layer_norm = LayerNorm(embed_dim, export=export) - self.pointwise_conv1 = nn.Conv1d(embed_dim, 2 * channels, kernel_size=1, stride=1, padding=0, bias=bias) - self.glu = nn.GLU(dim=1) - self.depthwise_conv = nn.Conv1d(channels, channels, depthwise_kernel_size, stride=1, padding=(depthwise_kernel_size - 1) // 2, groups=channels, bias=bias) - self.batch_norm = nn.BatchNorm1d(channels) - self.activation = get_activation_fn(activation_fn)(channels) - self.pointwise_conv2 = nn.Conv1d(channels, embed_dim, kernel_size=1, stride=1, padding=0, bias=bias) - self.dropout = nn.Dropout(dropout) - - def forward(self, x): - return self.dropout(self.pointwise_conv2(self.activation(self.batch_norm(self.depthwise_conv(self.glu(self.pointwise_conv1(self.layer_norm(x).transpose(1, 2)))))))).transpose(1, 2) - -def rotate_half(x): - x1, x2 = x[..., : x.shape[-1] // 2], x[..., x.shape[-1] // 2 :] - - return torch.cat((-x2, x1), dim=x1.ndim - 1) - -def apply_rotary_pos_emb(q, k, cos, sin, offset: int = 0): - cos, sin = (cos[offset : q.shape[0] + offset, ...], sin[offset : q.shape[0] + offset, ...]) - - return (q * cos) + (rotate_half(q) * sin), (k * cos) + (rotate_half(k) * sin) - -class RotaryPositionalEmbedding(nn.Module): - def __init__( - self, - dim, - base=10000, - precision=torch.half - ): - super().__init__() - inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2).float() / dim)) - self.register_buffer("inv_freq", inv_freq) - self.seq_len_cached = 0 - self.cos_cached = torch.empty(self.seq_len_cached, 1, 1, dim) - self.sin_cached = torch.empty(self.seq_len_cached, 1, 1, dim) - self.precision = precision - - def forward(self, x, seq_len = 0): - if seq_len > self.seq_len_cached: - self.seq_len_cached = seq_len - freqs = torch.einsum("i,j->ij", torch.arange(seq_len, device=x.device).type_as(self.inv_freq), self.inv_freq) - emb = torch.cat((freqs, freqs), dim=-1).to(x.device) - self.cos_cached = emb.cos().view(emb.size(0), 1, 1, emb.size(1)) - self.sin_cached = emb.sin().view(emb.size(0), 1, 1, emb.size(1)) - return self.cos_cached, self.sin_cached - -class ESPNETMultiHeadedAttention(nn.Module): - def __init__( - self, - n_feat, - n_head, - dropout - ): - super(ESPNETMultiHeadedAttention, self).__init__() - assert n_feat % n_head == 0 - self.d_k = n_feat // n_head - self.h = n_head - self.linear_q = nn.Linear(n_feat, n_feat) - self.linear_k = nn.Linear(n_feat, n_feat) - self.linear_v = nn.Linear(n_feat, n_feat) - self.linear_out = nn.Linear(n_feat, n_feat) - self.attn = None - self.dropout = nn.Dropout(p=dropout) - - def forward_qkv(self, query, key, value, **kwargs): - n_batch = query.size(0) - - return ( - self.linear_q(query).view(n_batch, -1, self.h, self.d_k).transpose(1, 2), - self.linear_k(key).view(n_batch, -1, self.h, self.d_k).transpose(1, 2), - self.linear_v(value).view(n_batch, -1, self.h, self.d_k).transpose(1, 2) - ) - - def forward_attention(self, value, scores, mask): - n_batch = value.size(0) - - if mask is not None: - scores = scores.masked_fill(mask.unsqueeze(1).unsqueeze(2).to(bool), float("-inf")) - self.attn = scores.softmax(dim=-1) - else: self.attn = scores.softmax(dim=-1) - - return self.linear_out(((self.dropout(self.attn) @ value).transpose(1, 2).contiguous().view(n_batch, -1, self.h * self.d_k))) - - def forward(self, query, key, value, key_padding_mask=None, **kwargs): - q, k, v = self.forward_qkv(query.transpose(0, 1), key.transpose(0, 1), value.transpose(0, 1)) - return self.forward_attention(v, (q @ k.transpose(-2, -1)) / math.sqrt(self.d_k), key_padding_mask).transpose(0, 1), None - -class RelPositionMultiHeadedAttention(ESPNETMultiHeadedAttention): - def __init__(self, n_feat, n_head, dropout, zero_triu=False): - super().__init__(n_feat, n_head, dropout) - self.zero_triu = zero_triu - self.linear_pos = nn.Linear(n_feat, n_feat, bias=False) - self.pos_bias_u = nn.Parameter(torch.zeros(self.h, self.d_k)) - self.pos_bias_v = nn.Parameter(torch.zeros(self.h, self.d_k)) - nn.init.xavier_uniform_(self.pos_bias_u) - nn.init.xavier_uniform_(self.pos_bias_v) - - def rel_shift(self, x): - x = torch.cat( - [ - torch.zeros( - (*x.size()[:3], 1), - device=x.device, - dtype=x.dtype - ), - x - ], - dim=-1 - ).view(*x.size()[:2], x.size(3) + 1, x.size(2))[:, :, 1:].view_as(x)[:, :, :, : x.size(-1) // 2 + 1] - - if self.zero_triu: x *= torch.ones((x.size(2), x.size(3)), device=x.device).tril(x.size(3) - x.size(2))[None, None, :, :] - return x - - def forward(self, query, key, value, pos_emb, key_padding_mask=None, **kwargs): - pos_emb = pos_emb.transpose(0, 1) - q, k, v = self.forward_qkv(query.transpose(0, 1), key.transpose(0, 1), value.transpose(0, 1)) - q = q.transpose(1, 2) - - return self.forward_attention( - v, ( - ( - (q + self.pos_bias_u).transpose(1, 2) @ k.transpose(-2, -1) - ) + self.rel_shift( - ((q + self.pos_bias_v).transpose(1, 2) @ self.linear_pos(pos_emb).view(pos_emb.size(0), -1, self.h, self.d_k).transpose(1, 2).transpose(-2, -1)) - ) - ) / math.sqrt(self.d_k), key_padding_mask).transpose(0, 1), None - -class RotaryPositionMultiHeadedAttention(ESPNETMultiHeadedAttention): - def __init__( - self, - n_feat, - n_head, - dropout, - precision, - rotary_emd_base=10000 - ): - super().__init__(n_feat, n_head, dropout) - precision = torch.float - self.rotary_ndims = self.d_k - if precision == "fp16": precision = torch.half - self.rotary_emb = RotaryPositionalEmbedding(self.rotary_ndims, base=rotary_emd_base, precision=precision) - - def forward(self, query, key, value, key_padding_mask=None, **kwargs): - T, B, _ = value.size() - - query = query.view(T, B, self.h, self.d_k) - key = key.view(T, B, self.h, self.d_k) - - value = value.view(T, B, self.h, self.d_k) - cos, sin = self.rotary_emb(value, seq_len=T) - - query, key = apply_rotary_pos_emb(query, key, cos, sin, offset=0) - query = query.view(T, B, self.h * self.d_k) - - key = key.view(T, B, self.h * self.d_k) - value = value.view(T, B, self.h * self.d_k) - q, k, v = self.forward_qkv(query.transpose(0, 1), key.transpose(0, 1), value.transpose(0, 1)) - - return self.forward_attention(v, (q @ k.transpose(-2, -1)) / math.sqrt(self.d_k), key_padding_mask).transpose(0, 1), None - -class ConformerEncoderLayer(nn.Module): - def __init__(self, embed_dim, ffn_embed_dim, attention_heads, dropout, use_fp16, depthwise_conv_kernel_size=31, activation_fn="swish", attn_type=None, pos_enc_type="abs"): - self.pos_enc_type = pos_enc_type - super(ConformerEncoderLayer, self).__init__() - self.ffn1 = FeedForwardModule(embed_dim, ffn_embed_dim, dropout, dropout) - self.self_attn_layer_norm = LayerNorm(embed_dim, export=False) - self.self_attn_dropout = nn.Dropout(dropout) - if attn_type == "espnet": - if self.pos_enc_type == "rel_pos": self.self_attn = RelPositionMultiHeadedAttention(embed_dim, attention_heads, dropout=dropout) - elif self.pos_enc_type == "rope": self.self_attn = RotaryPositionMultiHeadedAttention(embed_dim, attention_heads, dropout=dropout, precision=use_fp16) - elif self.pos_enc_type == "abs": self.self_attn = ESPNETMultiHeadedAttention(embed_dim, attention_heads, dropout=dropout) - else: raise Exception - else: self.self_attn = MultiheadAttention(embed_dim, attention_heads, dropout=dropout) - self.conv_module = ConvolutionModule(embed_dim=embed_dim, channels=embed_dim, depthwise_kernel_size=depthwise_conv_kernel_size, dropout=dropout, activation_fn=activation_fn) - self.ffn2 = FeedForwardModule(embed_dim, ffn_embed_dim, dropout, dropout, activation_fn=activation_fn) - self.final_layer_norm = LayerNorm(embed_dim, export=False) - - def forward(self, x, encoder_padding_mask, position_emb = None): - residual = x - x = self.ffn1(x) * 0.5 + residual - residual = x - - x = self.self_attn_layer_norm(x) - if self.pos_enc_type == "rel_pos": x, attn = self.self_attn(query=x, key=x, value=x, key_padding_mask=encoder_padding_mask, pos_emb=position_emb, need_weights=False) - else: x, attn = self.self_attn(query=x, key=x, value=x, key_padding_mask=encoder_padding_mask, need_weights=False) - - x = self.self_attn_dropout(x) - x = x + residual - residual = x - - x = residual + self.conv_module(x.transpose(0, 1)).transpose(0, 1) - residual = x - x = self.ffn2(x) - - layer_result = x - x = self.final_layer_norm(x * 0.5 + residual) - - return x, (attn, layer_result) - -class ConformerWav2Vec2EncoderLayer(ConformerEncoderLayer): - def forward( - self, - x, - self_attn_mask=None, - self_attn_padding_mask=None, - need_weights=False, - att_args=None, - position_emb=None - ): - return super().forward( - x, - self_attn_padding_mask, - position_emb - ) - -class TransformerSentenceEncoderWithAdapterLayer(TransformerSentenceEncoderLayer): - def __init__( - self, - embedding_dim = 768, - ffn_embedding_dim = 3072, - num_attention_heads = 8, - dropout = 0.1, - attention_dropout = 0.1, - activation_dropout = 0.1, - activation_fn = "relu", - layer_norm_first = False, - adapter_num=201, - adapter_dim=64, - adapter_act_fn="relu" - ): - super().__init__( - embedding_dim=embedding_dim, - ffn_embedding_dim=ffn_embedding_dim, - num_attention_heads=num_attention_heads, - dropout=dropout, - attention_dropout=attention_dropout, - activation_dropout=activation_dropout, - activation_fn=activation_fn, - layer_norm_first=layer_norm_first - ) - self.adapter_num = adapter_num - self.adapter_dim = adapter_dim - self.adapter_layer = AdapterFast(adapter_num, self.embedding_dim, self.adapter_dim, adapter_act_fn) - - def forward(self, x, self_attn_mask=None, self_attn_padding_mask=None, need_weights=False, att_args=None, corpus_key=None): - x, (attn, layer_result) = super().forward(x=x, self_attn_mask=self_attn_mask, self_attn_padding_mask=self_attn_padding_mask, need_weights=need_weights, att_args=att_args) - assert corpus_key is not None - assert len(set(corpus_key)) == 1 - return x + self.adapter_layer(x, corpus_key[0]), (attn, layer_result) - -class TransposeLast(nn.Module): - def __init__(self, deconstruct_idx=None, tranpose_dim=-2): - super().__init__() - self.deconstruct_idx = deconstruct_idx - self.tranpose_dim = tranpose_dim - - def forward(self, x): - if self.deconstruct_idx is not None: x = x[self.deconstruct_idx] - return x.transpose(self.tranpose_dim, -1) - -class TransformerEncoder(nn.Module): - def build_encoder_layer(self, args, **kwargs): - if args.layer_type == "transformer": - layer = TransformerSentenceEncoderLayer( - embedding_dim=self.embedding_dim, - ffn_embedding_dim=args.encoder_ffn_embed_dim, - num_attention_heads=args.encoder_attention_heads, - dropout=self.dropout, - attention_dropout=args.attention_dropout, - activation_dropout=args.activation_dropout, - activation_fn=args.activation_fn, - layer_norm_first=args.layer_norm_first - ) - elif args.layer_type == "conformer": - layer = ConformerWav2Vec2EncoderLayer( - embed_dim=self.embedding_dim, - ffn_embed_dim=args.encoder_ffn_embed_dim, - attention_heads=args.encoder_attention_heads, - dropout=args.dropout, - depthwise_conv_kernel_size=args.depthwise_conv_kernel_size, - activation_fn="swish", - attn_type=args.attn_type, - use_fp16=args.fp16, - pos_enc_type="abs" - ) - elif args.layer_type == "trf_adp": - use_adp = False - if args.adp_trf_idx == "all": use_adp = True - else: - if kwargs.get("layer_idx", None) in list(range(*[int(g) for g in args.adp_trf_idx.split(":")])): use_adp = True - - layer = ( - TransformerSentenceEncoderWithAdapterLayer( - embedding_dim=self.embedding_dim, - ffn_embedding_dim=args.encoder_ffn_embed_dim, - num_attention_heads=args.encoder_attention_heads, - dropout=self.dropout, - attention_dropout=args.attention_dropout, - activation_dropout=args.activation_dropout, - activation_fn=args.activation_fn, - layer_norm_first=args.layer_norm_first, - adapter_num=args.adp_num, - adapter_dim=args.adp_dim, - adapter_act_fn=args.adp_act_fn - ) - ) if use_adp else ( - TransformerSentenceEncoderLayer( - embedding_dim=self.embedding_dim, - ffn_embedding_dim=args.encoder_ffn_embed_dim, - num_attention_heads=args.encoder_attention_heads, - dropout=self.dropout, - attention_dropout=args.attention_dropout, - activation_dropout=args.activation_dropout, - activation_fn=args.activation_fn, - layer_norm_first=args.layer_norm_first - ) - ) - - return layer - - def __init__(self, args): - super().__init__() - self.dropout = args.dropout - self.embedding_dim = args.encoder_embed_dim - self.required_seq_len_multiple = args.required_seq_len_multiple - pos_conv_depth = getattr(args, "pos_conv_depth", 1) - if pos_conv_depth > 1: - num_layers = args.pos_conv_depth - k = max(3, args.conv_pos // num_layers) - - def make_conv_block(e, k, g, l): - return nn.Sequential( - *[ - nn.Sequential( - nn.Conv1d( - e, - e, - kernel_size=k, - padding=k // 2, - groups=g - ), - SamePad(k), - TransposeLast(), - LayerNorm(e, elementwise_affine=False), - TransposeLast(), - nn.GELU() - ) - for _ in range(l) - ] - ) - - self.pos_conv = make_conv_block( - self.embedding_dim, - k, - args.conv_pos_groups, - num_layers - ) - else: - self.pos_conv = make_conv_pos( - self.embedding_dim, - args.conv_pos, - args.conv_pos_groups - ) - - self.layers = nn.ModuleList([self.build_encoder_layer(args, layer_idx=ii) for ii in range(args.encoder_layers)]) - self.layer_norm_first = args.layer_norm_first - self.layer_norm = LayerNorm(self.embedding_dim) - self.layerdrop = args.encoder_layerdrop - self.apply(init_bert_params) - - def forward(self, x, padding_mask=None, layer=None, corpus_key=None): - x, layer_results = self.extract_features(x, padding_mask, layer, corpus_key=corpus_key) - if self.layer_norm_first and layer is None: x = self.layer_norm(x) - return x, layer_results - - def extract_features(self, x, padding_mask=None, tgt_layer=None, min_layer=0, corpus_key=None): - if padding_mask is not None: x = index_put(x, padding_mask, 0) - x = x + self.pos_conv(x.transpose(1, 2)).transpose(1, 2) - if not self.layer_norm_first: x = self.layer_norm(x) - x, pad_length = pad_to_multiple(x, self.required_seq_len_multiple, dim=-2, value=0) - if pad_length > 0 and padding_mask is None: - padding_mask = x.new_zeros((x.size(0), x.size(1)), dtype=torch.bool) - padding_mask[:, -pad_length:] = True - else: padding_mask, _ = pad_to_multiple(padding_mask, self.required_seq_len_multiple, dim=-1, value=True) - x = F.dropout(x, p=self.dropout, training=self.training).transpose(0, 1) - layer_results = [] - r = None - - for i, layer in enumerate(self.layers): - dropout_probability = np.random.random() if self.layerdrop > 0 else 1 - - if not self.training or (dropout_probability > self.layerdrop): - layer_check = layer - - if (corpus_key is None) or (not isinstance(layer_check, (TransformerSentenceEncoderWithAdapterLayer))): - x, (z, lr) = layer( - x, - self_attn_padding_mask=padding_mask, - need_weights=False - ) - else: - x, (z, lr) = layer( - x, - self_attn_padding_mask=padding_mask, - need_weights=False, - corpus_key=corpus_key - ) - - if i >= min_layer: layer_results.append((x, z, lr)) - - if i == tgt_layer: - r = x - break - - if r is not None: x = r - x = x.transpose(0, 1) - - if pad_length > 0: - x = x[:, :-pad_length] - - def undo_pad(a, b, c): - return (a[:-pad_length], b[:-pad_length] if b is not None else b, c[:-pad_length]) - - layer_results = [undo_pad(*u) for u in layer_results] - - return x, layer_results - - def max_positions(self): - return self.args.max_positions - - def upgrade_state_dict_named(self, state_dict, name): - return state_dict - -class Fp32GroupNorm(nn.GroupNorm): - def __init__( - self, - *args, - **kwargs - ): - super().__init__(*args, **kwargs) - - def forward(self, input): - output = F.group_norm( - input.float(), - self.num_groups, - self.weight.float() if self.weight is not None else None, - self.bias.float() if self.bias is not None else None, - self.eps - ) - - return output.type_as(input) - -class Fp32LayerNorm(nn.LayerNorm): - def __init__( - self, - *args, - **kwargs - ): - super().__init__(*args, **kwargs) - - def forward(self, input): - output = F.layer_norm( - input.float(), - self.normalized_shape, - self.weight.float() if self.weight is not None else None, - self.bias.float() if self.bias is not None else None, - self.eps - ) - - return output.type_as(input) - -class ConvFeatureExtractionModel(nn.Module): - def __init__( - self, - conv_layers, - dropout = 0.0, - mode = "default", - conv_bias = False - ): - super().__init__() - assert mode in {"default", "layer_norm"} - - def block(n_in, n_out, k, stride, is_layer_norm=False, is_group_norm=False, conv_bias=False): - def make_conv(): - conv = nn.Conv1d(n_in, n_out, k, stride=stride, bias=conv_bias) - nn.init.kaiming_normal_(conv.weight) - return conv - - assert (is_layer_norm and is_group_norm) == False - - if is_layer_norm: - return nn.Sequential( - make_conv(), - nn.Dropout(p=dropout), - nn.Sequential( - TransposeLast(), - Fp32LayerNorm( - dim, - elementwise_affine=True - ), - TransposeLast() - ), - nn.GELU() - ) - elif is_group_norm: - return nn.Sequential( - make_conv(), - nn.Dropout(p=dropout), - Fp32GroupNorm(dim, dim, affine=True), - nn.GELU() - ) - else: - return nn.Sequential( - make_conv(), - nn.Dropout(p=dropout), - nn.GELU() - ) - - in_d = 1 - self.conv_layers = nn.ModuleList() - for i, cl in enumerate(conv_layers): - assert len(cl) == 3 - (dim, k, stride) = cl - - self.conv_layers.append( - block( - in_d, - dim, - k, - stride, - is_layer_norm=mode == "layer_norm", - is_group_norm=mode == "default" and i == 0, - conv_bias=conv_bias - ) - ) - - in_d = dim - - def forward(self, x): - x = x.unsqueeze(1) - for conv in self.conv_layers: - x = conv(x) - - return x - -class GradMultiply(torch.autograd.Function): - @staticmethod - def forward( - ctx, - x, - scale - ): - ctx.scale = scale - res = x.new(x) - return res - - @staticmethod - def backward(ctx, grad): - return grad * ctx.scale, None - -class BaseFairseqModel(nn.Module): - def __init__(self): - super().__init__() - self._is_generation_fast = False - - def get_targets(self, sample, net_output): - return sample["target"] - - def extract_features(self, *args, **kwargs): - return self(*args, **kwargs) - - def load_state_dict(self, state_dict, strict=True, model_cfg = None, args = None): - self.upgrade_state_dict(state_dict) - new_state_dict = prune_state_dict(state_dict, model_cfg) - return super().load_state_dict(new_state_dict, strict) - - def upgrade_state_dict(self, state_dict): - self.upgrade_state_dict_named(state_dict, "") - - def upgrade_state_dict_named(self, state_dict, name): - assert state_dict is not None - - def do_upgrade(m, prefix): - if len(prefix) > 0: prefix += "." - for n, c in m.named_children(): - name = prefix + n - if hasattr(c, "upgrade_state_dict_named"): c.upgrade_state_dict_named(state_dict, name) - elif hasattr(c, "upgrade_state_dict"): c.upgrade_state_dict(state_dict) - do_upgrade(c, name) - - do_upgrade(self, name) - - def make_generation_fast_(self, **kwargs): - if self._is_generation_fast: return - self._is_generation_fast = True - - def apply_remove_weight_norm(module): - try: - if hasattr(module, "parametrizations") and "weight" in module.parametrizations: parametrize.remove_parametrizations(module, "weight", leave_parametrized=True) - else: nn.utils.remove_weight_norm(module) - except (AttributeError, ValueError): - return - - self.apply(apply_remove_weight_norm) - def apply_make_generation_fast_(module, prefix): - if len(prefix) > 0: prefix += "." - - base_func = BaseFairseqModel.make_generation_fast_ - for n, m in module.named_modules(): - if (m != self and hasattr(m, "make_generation_fast_") and m.make_generation_fast_.__func__ is not base_func): m.make_generation_fast_(name=prefix + n, **kwargs) - - apply_make_generation_fast_(self, "") - self.eval() - -class HubertConfig: - def __init__( - self, - _name = None, - label_rate = 50, - encoder_layers_1 = 3, - logit_temp_ctr = 0.1, - num_negatives = 100, - cross_sample_negatives = 0, - ctr_layers = [-6], - crop_seq_to_multiple = 1, - extractor_mode = "default", - encoder_layers = 12, - encoder_embed_dim = 768, - encoder_ffn_embed_dim = 3072, - encoder_attention_heads = 12, - activation_fn = "gelu", - layer_type = "transformer", - dropout = 0.1, - attention_dropout = 0.1, - activation_dropout = 0.0, - encoder_layerdrop = 0.0, - dropout_input = 0.0, - dropout_features = 0.0, - final_dim = 0, - untie_final_proj = False, - layer_norm_first = False, - conv_feature_layers = "[(512,10,5)] + [(512,3,2)] * 4 + [(512,2,2)] * 2", - conv_bias = False, - logit_temp = 0.1, - target_glu = False, - feature_grad_mult = 1.0, - mask_length = 10, - mask_prob = 0.65, - mask_selection = "static", - mask_other = 0.0, - no_mask_overlap = False, - mask_min_space = 1, - mask_channel_length = 10, - mask_channel_prob = 0.0, - mask_channel_selection = "static", - mask_channel_other = 0.0, - no_mask_channel_overlap = False, - mask_channel_min_space = 1, - conv_pos = 128, - conv_pos_groups = 16, - conv_pos_batch_norm = False, - latent_temp = (2, 0.5, 0.999995), - skip_masked = False, - skip_nomask = False, - checkpoint_activations = False, - required_seq_len_multiple = 2, - depthwise_conv_kernel_size = 31, - attn_type = "", - pos_enc_type = "abs", - fp16 = False - ): - self._name = _name - self.label_rate = label_rate - self.encoder_layers_1 = encoder_layers_1 - self.logit_temp_ctr = logit_temp_ctr - self.num_negatives = num_negatives - self.cross_sample_negatives = cross_sample_negatives - self.ctr_layers = ctr_layers - self.crop_seq_to_multiple = crop_seq_to_multiple - self.extractor_mode = extractor_mode - self.encoder_layers = encoder_layers - self.encoder_embed_dim = encoder_embed_dim - self.encoder_ffn_embed_dim = encoder_ffn_embed_dim - self.encoder_attention_heads = encoder_attention_heads - self.activation_fn = activation_fn - self.layer_type = layer_type - self.dropout = dropout - self.attention_dropout = attention_dropout - self.activation_dropout = activation_dropout - self.encoder_layerdrop = encoder_layerdrop - self.dropout_input = dropout_input - self.dropout_features = dropout_features - self.final_dim = final_dim - self.untie_final_proj = untie_final_proj - self.layer_norm_first = layer_norm_first - self.conv_feature_layers = conv_feature_layers - self.conv_bias = conv_bias - self.logit_temp = logit_temp - self.target_glu = target_glu - self.feature_grad_mult = feature_grad_mult - self.mask_length = mask_length - self.mask_prob = mask_prob - self.mask_selection = mask_selection - self.mask_other = mask_other - self.no_mask_overlap = no_mask_overlap - self.mask_min_space = mask_min_space - self.mask_channel_length = mask_channel_length - self.mask_channel_prob = mask_channel_prob - self.mask_channel_selection = mask_channel_selection - self.mask_channel_other = mask_channel_other - self.no_mask_channel_overlap = no_mask_channel_overlap - self.mask_channel_min_space = mask_channel_min_space - self.conv_pos = conv_pos - self.conv_pos_groups = conv_pos_groups - self.conv_pos_batch_norm = conv_pos_batch_norm - self.latent_temp = latent_temp - self.skip_masked = skip_masked - self.skip_nomask = skip_nomask - self.checkpoint_activations = checkpoint_activations - self.required_seq_len_multiple = required_seq_len_multiple - self.depthwise_conv_kernel_size = depthwise_conv_kernel_size - self.attn_type = attn_type - self.pos_enc_type = pos_enc_type - self.fp16 = fp16 - -class HubertModel(BaseFairseqModel): - def __init__( - self, - cfg, - num_classes - ): - super().__init__() - feature_enc_layers = eval(cfg.conv_feature_layers) - self.embed = feature_enc_layers[-1][0] - self.feature_extractor = ConvFeatureExtractionModel(conv_layers=feature_enc_layers, dropout=0.0, mode=cfg.extractor_mode, conv_bias=cfg.conv_bias) - feature_ds_rate = np.prod([s for _, _, s in feature_enc_layers]) - self.feat2tar_ratio = cfg.label_rate * feature_ds_rate / 16000 - self.post_extract_proj = (nn.Linear(self.embed, cfg.encoder_embed_dim) if self.embed != cfg.encoder_embed_dim else None) - self.mask_prob = cfg.mask_prob - self.mask_selection = cfg.mask_selection - self.mask_other = cfg.mask_other - self.mask_length = cfg.mask_length - self.no_mask_overlap = cfg.no_mask_overlap - self.mask_min_space = cfg.mask_min_space - self.mask_channel_prob = cfg.mask_channel_prob - self.mask_channel_selection = cfg.mask_channel_selection - self.mask_channel_other = cfg.mask_channel_other - self.mask_channel_length = cfg.mask_channel_length - self.no_mask_channel_overlap = cfg.no_mask_channel_overlap - self.mask_channel_min_space = cfg.mask_channel_min_space - self.dropout_input = nn.Dropout(cfg.dropout_input) - self.dropout_features = nn.Dropout(cfg.dropout_features) - self.feature_grad_mult = cfg.feature_grad_mult - self.logit_temp = cfg.logit_temp - self.skip_masked = cfg.skip_masked - self.skip_nomask = cfg.skip_nomask - final_dim = cfg.final_dim if cfg.final_dim > 0 else cfg.encoder_embed_dim - self.mask_emb = nn.Parameter(torch.FloatTensor(cfg.encoder_embed_dim).uniform_()) - self.encoder = TransformerEncoder(cfg) - self.layer_norm = LayerNorm(self.embed) - self.target_glu = None - if cfg.target_glu: self.target_glu = nn.Sequential(nn.Linear(final_dim, final_dim * 2), nn.GLU()) - self.untie_final_proj = cfg.untie_final_proj - self.final_proj = nn.Linear(cfg.encoder_embed_dim, final_dim) - self.num_classes = [num_classes] - self.label_embs_concat = nn.Parameter(torch.FloatTensor(sum(self.num_classes), final_dim)) - nn.init.uniform_(self.label_embs_concat) - - def upgrade_state_dict_named(self, state_dict, name): - super().upgrade_state_dict_named(state_dict, name) - return state_dict - - def apply_mask(self, x, padding_mask, target_list): - B, T, C = x.shape - - if self.mask_prob > 0: - mask_indices = torch.from_numpy( - compute_mask_indices( - (B, T), - padding_mask, - self.mask_prob, - self.mask_length, - self.mask_selection, - self.mask_other, - min_masks=2, - no_overlap=self.no_mask_overlap, - min_space=self.mask_min_space - ) - ).to(x.device) - - x[mask_indices] = self.mask_emb - else: mask_indices = None - - if self.mask_channel_prob > 0: - x[( - torch.from_numpy( - compute_mask_indices( - (B, C), - None, - self.mask_channel_prob, - self.mask_channel_length, - self.mask_channel_selection, - self.mask_channel_other, - no_overlap=self.no_mask_channel_overlap, - min_space=self.mask_channel_min_space - ) - ).to(x.device).unsqueeze(1).expand(-1, T, -1) - )] = 0 - - return x, mask_indices - - def compute_nce(self, x, pos, negs): - neg_is_pos = (pos == negs).all(-1) - - logits = torch.cosine_similarity( - x.float(), - torch.cat( - [pos.unsqueeze(0), negs], - dim=0 - ).float(), - dim=-1 - ).type_as(x) - logits /= self.logit_temp - - if neg_is_pos.any(): logits[1:][neg_is_pos] = float("-inf") - return logits.transpose(0, 1) - - def forward_features(self, source): - if self.feature_grad_mult > 0: - features = self.feature_extractor(source) - if self.feature_grad_mult != 1.0: features = GradMultiply.apply(features, self.feature_grad_mult) - else: - with torch.no_grad(): - features = self.feature_extractor(source) - return features - - def forward_targets(self, features, target_list): - feat_tsz = features.size(2) - targ_tsz = min([t.size(1) for t in target_list]) - if self.feat2tar_ratio * feat_tsz > targ_tsz: - feat_tsz = int(targ_tsz / self.feat2tar_ratio) - features = features[..., :feat_tsz] - - return features, [t[:, (torch.arange(feat_tsz).float() * self.feat2tar_ratio).long()] for t in target_list] - - def forward_padding_mask(self, features, padding_mask): - extra = padding_mask.size(1) % features.size(1) - if extra > 0: padding_mask = padding_mask[:, :-extra] - return padding_mask.view(padding_mask.size(0), features.size(1), -1).all(-1) - - def forward(self, source, target_list = None, padding_mask = None, mask = True, features_only = False, output_layer = None): - features = self.forward_features(source) - if target_list is not None: features, target_list = self.forward_targets(features, target_list) - - features_pen = features.float().pow(2).mean() - features = self.layer_norm(features.transpose(1, 2)) - unmasked_features = features.clone() - - if padding_mask is not None: padding_mask = self.forward_padding_mask(features, padding_mask) - if self.post_extract_proj is not None: features = self.post_extract_proj(features) - - features = self.dropout_input(features) - unmasked_features = self.dropout_features(unmasked_features) - - if mask: x, mask_indices = self.apply_mask(features, padding_mask, target_list) - else: x, mask_indices = features, None - - x, _ = self.encoder(x, padding_mask=padding_mask, layer=None if output_layer is None else output_layer - 1) - if features_only: return {"x": x, "padding_mask": padding_mask, "features": features} - - def compute_pred(proj_x, target, label_embs): - y = torch.index_select(label_embs, 0, target.long()) - negs = label_embs.unsqueeze(1).expand(-1, proj_x.size(0), -1) - - if self.target_glu: - y = self.target_glu(y) - negs = self.target_glu(negs) - - return self.compute_nce(proj_x, y, negs) - - label_embs_list = self.label_embs_concat.split(self.num_classes, 0) - if not self.skip_masked: - masked_indices = torch.logical_and(~padding_mask, mask_indices) - proj_x_m = self.final_proj(x[masked_indices]) - - logit_m_list = [ - compute_pred( - proj_x_m, - t[masked_indices], - label_embs_list[i] - ) - for i, (proj_x_m, t) in enumerate(zip( - proj_x_m.chunk(len(target_list), dim=-1) if self.untie_final_proj else [proj_x_m for _ in range(len(target_list))], - target_list - )) - ] - else: logit_m_list = [None for _ in target_list] - - if not self.skip_nomask: - nomask_indices = torch.logical_and(~padding_mask, ~mask_indices) - proj_x_u = self.final_proj(x[nomask_indices]) - - logit_u_list = [ - compute_pred( - proj_x_u, - t[nomask_indices], - label_embs_list[i] - ) - for i, (proj_x_u, t) in enumerate(zip( - proj_x_u.chunk(len(target_list), dim=-1) if self.untie_final_proj else [proj_x_u for _ in range(len(target_list))], - target_list - )) - ] - else: logit_u_list = [None for _ in target_list] - - return {"logit_m_list": logit_m_list, "logit_u_list": logit_u_list, "padding_mask": padding_mask, "features_pen": features_pen} - - def extract_features(self, source, padding_mask = None, mask = False, ret_conv = False, output_layer = None): - if padding_mask is None: padding_mask = torch.BoolTensor(source.shape).fill_(False).to(source.device) - res = self.forward(source, padding_mask=padding_mask, mask=mask, features_only=True, output_layer=output_layer) - return res["features"] if ret_conv else res["x"], res["padding_mask"] - - def get_logits(self, net_output, is_masked=True): - return [x.float() for x in (net_output["logit_m_list"] if is_masked else net_output["logit_u_list"]) if x is not None] - - def get_targets(self, net_output, is_masked=True): - return [x.new_zeros(x.size(0), dtype=torch.long) for x in self.get_logits(net_output, is_masked)] - - def get_extra_losses(self, net_output): - extra_losses, names = [], [] - if "features_pen" in net_output: - extra_losses.append(net_output["features_pen"]) - names.append("features_pen") - - return extra_losses, names - - def remove_pretraining_modules(self): - self.target_glu = None - self.final_proj = None \ No newline at end of file diff --git a/main/library/embedders/onnx.py b/main/library/embedders/onnx.py deleted file mode 100644 index 6674bcd3d1c13446d63031ede1e8ee038306dfef..0000000000000000000000000000000000000000 --- a/main/library/embedders/onnx.py +++ /dev/null @@ -1,44 +0,0 @@ -import torch -import onnxruntime - -import numpy as np - -class HubertModelONNX: - def __init__( - self, - embedder_model_path, - providers - ): - sess_options = onnxruntime.SessionOptions() - sess_options.log_severity_level = 3 - - self.model = onnxruntime.InferenceSession( - embedder_model_path, - sess_options=sess_options, - providers=providers - ) - - self._finalproj = False - self.final_proj = self._final_proj - - def _final_proj(self, source): - return source - - def extract_features(self, source, output_layer = None): - device = source.device - - logits = self.model.run( - [self.model.get_outputs()[0].name, self.model.get_outputs()[1].name], - { - self.model.get_inputs()[0].name: source.float().detach().cpu().numpy(), - self.model.get_inputs()[1].name: np.array(output_layer, dtype=np.int64), - } - ) - - return [ - torch.as_tensor( - logits[int(self._finalproj)], - dtype=torch.float32, - device=device - ) - ] \ No newline at end of file diff --git a/main/library/embedders/ppg.py b/main/library/embedders/ppg.py deleted file mode 100644 index ce5ea0fc06cd148d9b0e394fc66dbc1153815c33..0000000000000000000000000000000000000000 --- a/main/library/embedders/ppg.py +++ /dev/null @@ -1,69 +0,0 @@ -import os -import sys -import torch - -from contextlib import nullcontext - -sys.path.append(os.getcwd()) - -from main.app.variables import config -from main.library.speaker_diarization.whisper import Whisper, ModelDimensions, log_mel_spectrogram, pad_or_trim - -class WhisperModel(torch.nn.Module): - def __init__( - self, - model_path - ): - super().__init__() - checkpoint = torch.load(model_path, map_location="cpu", weights_only=True) - dims = ModelDimensions(**checkpoint["dims"]) - self._final_proj = torch.nn.Linear(dims.n_text_state, 768) - self.final_proj = torch.nn.Linear(768, 256) - self.model = Whisper(dims) - self.model.load_state_dict(checkpoint["model_state_dict"]) - del self.model.decoder - self.rank = 0 - - def forward(self, audio): - ppgln = audio.shape[1] // 320 - - mel = log_mel_spectrogram( - pad_or_trim(audio[0]) - ).to(audio.device) - - autocast_enabled = config.is_half and audio.device.type in ["cuda", "xpu"] - autocast_dtype = ( - torch.float32 - if not autocast_enabled else - torch.float16 - ) - - autocasts = torch.amp.autocast( - audio.device.type, - enabled=autocast_enabled, - dtype=autocast_dtype - ) if not audio.device.type.startswith(("ocl", "privateuseone")) else nullcontext() - - with torch.no_grad(): - with autocasts: - ppg_raw = self.model.encoder(mel.unsqueeze(0)) - ppg_projected = self._final_proj(ppg_raw) - - ppg = ppg_projected.data.float() - ppg = ppg[:, :ppgln, :] - - return [ppg] - - def extract_features(self, source, output_layer = None): - if self.rank == 0: - self.rank += 1 - - self.model.to(source.device).eval() - self.model.encoder.to(source.device).eval() - if config.compile_all: self.model.encoder = torch.compile(self.model.encoder, mode=config.compile_mode) - - if config.is_half: - self.model.half() - self.model.encoder.half() - - return self.forward(source) \ No newline at end of file diff --git a/main/library/embedders/transformers.py b/main/library/embedders/transformers.py deleted file mode 100644 index b88d56de5f30f8ef5692d056951340a244349de3..0000000000000000000000000000000000000000 --- a/main/library/embedders/transformers.py +++ /dev/null @@ -1,16 +0,0 @@ -from torch import nn -from transformers import HubertModel - -class HubertModelWithFinalProj(HubertModel): - def __init__(self, config): - super().__init__(config) - self.final_proj = nn.Linear( - config.hidden_size, - config.classifier_proj_size - ) - - def extract_features(self, source, output_layer=None): - outputs = self.forward(source, output_hidden_states=True, return_dict=True) - feats = outputs.last_hidden_state if output_layer is None else outputs.hidden_states[output_layer] - - return [feats] \ No newline at end of file diff --git a/main/library/generators/bigvgan.py b/main/library/generators/bigvgan.py deleted file mode 100644 index 67a79ade0e05ca26ab4e158f22b4f9093e084eca..0000000000000000000000000000000000000000 --- a/main/library/generators/bigvgan.py +++ /dev/null @@ -1,439 +0,0 @@ -import math -import torch - -import numpy as np -import torch.nn as nn -import torch.nn.functional as F -import torch.nn.utils.parametrize as parametrize - -from torch.nn.utils import remove_weight_norm -from torch.nn.utils.parametrizations import weight_norm - -class AntiAliasActivation(nn.Module): - def __init__( - self, - channels, - up=2, - down=2, - up_k=12, - down_k=12 - ): - super().__init__() - self.up = UpSample1d(up, up_k) - self.act = SnakeBeta(channels) - self.down = DownSample1d(down, down_k) - - def forward(self, x): - return self.down( - self.act(self.up(x)) - ) - -class SnakeBeta(nn.Module): - def __init__( - self, - channels - ): - super().__init__() - self.alpha = nn.Parameter(torch.zeros(1, channels, 1)) - self.beta = nn.Parameter(torch.zeros(1, channels, 1)) - - def forward(self, x): - return x + (1.0 / (self.beta.exp() + 1e-9)) * (x * self.alpha.exp()).sin().pow(2) - -def kaiser_sinc_filter1d( - cutoff, - half_width, - kernel_size -): - even = kernel_size % 2 == 0 - half_size = kernel_size // 2 - delta_f = 4 * half_width - - A = 2.285 * (half_size - 1) * math.pi * delta_f + 7.95 - - if A > 50.0: - beta = 0.1102 * (A - 8.7) - elif A >= 21.0: - beta = 0.5842 * (A - 21) ** 0.4 + 0.07886 * (A - 21.0) - else: - beta = 0.0 - - window = torch.kaiser_window( - kernel_size, - beta=beta, - periodic=False - ) - - time = ( - torch.arange(-half_size, half_size) + 0.5 - ) if even else ( - torch.arange(kernel_size) - half_size - ) - - if cutoff == 0: - filter = torch.zeros_like(time) - else: - filter = 2 * cutoff * window * torch.sinc(2 * cutoff * time) - filter /= filter.sum() - - return filter.view(1, 1, kernel_size) - -class UpSample1d(nn.Module): - def __init__( - self, - ratio=2, - kernel_size=None - ): - super().__init__() - self.ratio = ratio - self.stride = ratio - - kernel_size = int(6 * ratio // 2) * 2 if kernel_size is None else kernel_size - - self.pad = kernel_size // ratio - 1 - self.pad_left = self.pad * self.stride + (kernel_size - self.stride) // 2 - self.pad_right = self.pad * self.stride + (kernel_size - self.stride + 1) // 2 - - filter = kaiser_sinc_filter1d( - cutoff=0.5 / ratio, - half_width=0.6 / ratio, - kernel_size=kernel_size - ) - self.register_buffer("filter", filter) - - def forward(self, x): - x = self.ratio * F.conv_transpose1d( - F.pad( - x, - (self.pad, self.pad), - mode="replicate" - ), - self.filter.expand(x.size(1), -1, -1), - stride=self.stride, - groups=x.size(1) - ) - - return x[..., self.pad_left : -self.pad_right] - -class LowPassFilter1d(nn.Module): - def __init__( - self, - cutoff=0.5, - half_width=0.6, - stride=1, - kernel_size=12 - ): - super().__init__() - if cutoff < -0.0 or cutoff > 0.5: - raise ValueError - - even = kernel_size % 2 == 0 - self.pad_left = kernel_size // 2 - int(even) - self.pad_right = kernel_size // 2 - self.stride = stride - - filter = kaiser_sinc_filter1d( - cutoff, - half_width, - kernel_size - ) - self.register_buffer("filter", filter) - - def forward(self, x): - return F.conv1d( - F.pad( - x, - (self.pad_left, self.pad_right), - mode="replicate" - ), - self.filter.expand(x.size(1), -1, -1), - stride=self.stride, - groups=x.size(1) - ) - -class DownSample1d(nn.Module): - def __init__( - self, - ratio=2, - kernel_size=None - ): - super().__init__() - self.lowpass = LowPassFilter1d( - cutoff=0.5 / ratio, - half_width=0.6 / ratio, - stride=ratio, - kernel_size=int(6 * ratio // 2) * 2 if kernel_size is None else kernel_size, - ) - - def forward(self, x): - return self.lowpass(x) - -class AMPLayer(nn.Module): - def __init__( - self, - channels, - kernel_size, - dilation - ): - super().__init__() - self.conv1 = weight_norm( - nn.Conv1d( - channels, - channels, - kernel_size, - padding=(kernel_size * dilation - dilation) // 2, - dilation=dilation, - ) - ) - self.conv2 = weight_norm( - nn.Conv1d( - channels, - channels, - kernel_size, - padding=kernel_size // 2, - dilation=1 - ) - ) - - self.act1 = AntiAliasActivation(channels) - self.act2 = AntiAliasActivation(channels) - - def forward(self, x): - y = self.conv1(self.act1(x)) - y = self.conv2(self.act2(y)) - - return x + y - - def remove_weight_norm(self): - if hasattr(self.conv1, "parametrizations") and "weight" in self.conv1.parametrizations: parametrize.remove_parametrizations(self.conv1, "weight", leave_parametrized=True) - else: remove_weight_norm(self.conv1) - - if hasattr(self.conv2, "parametrizations") and "weight" in self.conv2.parametrizations: parametrize.remove_parametrizations(self.conv2, "weight", leave_parametrized=True) - else: remove_weight_norm(self.conv2) - -class AMPBlock(nn.Module): - def __init__( - self, - channels, - kernel_size, - dilations - ): - super().__init__() - self.layers = nn.ModuleList([ - AMPLayer(channels, kernel_size, dilation) - for dilation in dilations - ]) - - def forward(self, x): - for layer in self.layers: - x = layer(x) - - return x - - def remove_weight_norm(self): - for layer in self.layers: - layer.remove_weight_norm() - -class SineGen(nn.Module): - def __init__( - self, - sampling_rate, - harmonic_num=0, - sine_amp=0.1, - noise_std=0.003, - voiced_threshold=0, - flag_for_pulse=False, - ): - super(SineGen, self).__init__() - self.sine_amp = sine_amp - self.noise_std = noise_std - self.harmonic_num = harmonic_num - self.dim = self.harmonic_num + 1 - self.sampling_rate = sampling_rate - self.voiced_threshold = voiced_threshold - self.flag_for_pulse = flag_for_pulse - - def _f02uv(self, f0): - return (f0 > self.voiced_threshold).float() - - def _f02sine(self, f0_values): - rad_values = (f0_values / self.sampling_rate) % 1 - rand_ini = torch.rand(f0_values.shape[0], f0_values.shape[2], dtype=f0_values.dtype, device=f0_values.device) - - rand_ini[:, 0] = 0 - rad_values[:, 0, :] = rad_values[:, 0, :] + rand_ini - - tmp_over_one = torch.cumsum(rad_values, 1) % 1 - tmp_over_one_idx = (tmp_over_one[:, 1:, :] - tmp_over_one[:, :-1, :]) < 0 - - cumsum_shift = torch.zeros_like(rad_values) - cumsum_shift[:, 1:, :] = tmp_over_one_idx * -1.0 - - return (torch.cumsum(rad_values + cumsum_shift, dim=1) * 2 * np.pi).sin() - - def forward(self, f0): - with torch.no_grad(): - f0_buf = torch.zeros(f0.shape[0], f0.shape[1], self.dim, dtype=f0.dtype, device=f0.device) - f0_buf[:, :, 0] = f0[:, :, 0] - - for idx in np.arange(self.harmonic_num): - f0_buf[:, :, idx + 1] = f0_buf[:, :, 0] * (idx + 2) - - sine_waves = self._f02sine(f0_buf) * self.sine_amp - uv = self._f02uv(f0) - sine_waves = sine_waves * uv + ((uv * self.noise_std + (1 - uv) * self.sine_amp / 3) * torch.randn_like(sine_waves)) - - return sine_waves - -class SourceModuleHnNSF(nn.Module): - def __init__( - self, - sampling_rate, - harmonic_num=0, - sine_amp=0.1, - add_noise_std=0.003, - voiced_threshod=0, - ): - super(SourceModuleHnNSF, self).__init__() - self.sine_amp = sine_amp - self.noise_std = add_noise_std - self.l_sin_gen = SineGen(sampling_rate, harmonic_num, sine_amp, add_noise_std, voiced_threshod) - self.l_linear = nn.Linear(harmonic_num + 1, 1) - self.l_tanh = nn.Tanh() - - def forward(self, x): - sine_merge = self.l_tanh( - self.l_linear( - self.l_sin_gen(x).to(dtype=self.l_linear.weight.dtype) - ) - ) - - return sine_merge - -class BigVGANGenerator(nn.Module): - def __init__( - self, - in_channel, - upsample_initial_channel, - upsample_rates, - upsample_kernel_sizes, - resblock_kernel_sizes, - resblock_dilations, - gin_channels, - sample_rate, - harmonic_num, - ): - super().__init__() - self.num_kernels = len(resblock_kernel_sizes) - self.num_upsamples = len (upsample_rates) - self.f0_upsample = nn.Upsample(scale_factor=np.prod(upsample_rates)) - self.m_source = SourceModuleHnNSF(sample_rate, harmonic_num) - - self.conv_pre = weight_norm( - nn.Conv1d( - in_channel, - upsample_initial_channel, - kernel_size=7, - stride=1, - padding=3 - ) - ) - - self.amps = nn.ModuleList() - self.upsamples = nn.ModuleList() - self.noise_convs = nn.ModuleList() - - stride_f0s = [ - math.prod(upsample_rates[i + 1 :]) if i + 1 < self.num_upsamples else 1 - for i in range(self.num_upsamples) - ] - - for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)): - padding = ((k - u) // 2) if u % 2 == 0 else (u // 2 + u % 2) - - self.upsamples.append( - weight_norm( - nn.ConvTranspose1d( - upsample_initial_channel // (2**i), - upsample_initial_channel // (2 ** (i + 1)), - kernel_size=k, - stride=u, - padding=padding, - output_padding=u % 2, - ) - ) - ) - - stride = stride_f0s[i] - kernel = (1 if stride == 1 else stride * 2 - stride % 2) - padding = (0 if stride == 1 else (kernel - stride) // 2) - - self.noise_convs.append( - nn.Conv1d( - 1, - upsample_initial_channel // (2 ** (i + 1)), - kernel_size=kernel, - stride=stride, - padding=padding, - ) - ) - - for i in range(len(self.upsamples)): - channel = upsample_initial_channel // (2 ** (i + 1)) - - self.amps.append( - nn.ModuleList([ - AMPBlock(channel, kernel_size=k, dilations=d) - for k, d in zip(resblock_kernel_sizes, resblock_dilations) - ]) - ) - - self.act_post = AntiAliasActivation(channel) - self.conv_post = weight_norm( - nn.Conv1d( - channel, - 1, - kernel_size=7, - stride=1, - padding=3 - ) - ) - - if gin_channels != 0: - self.cond = nn.Conv1d( - gin_channels, - upsample_initial_channel, - 1 - ) - - def forward(self, x, f0, g=None): - har_source = self.m_source(self.f0_upsample(f0[:, None, :]).transpose(-1, -2)).transpose(-1, -2) - x = self.conv_pre(x) - if g is not None: x += self.cond(g) - - for up, amp, noise_conv in zip(self.upsamples, self.amps, self.noise_convs): - xs = 0 - - x = up(x) - x += noise_conv(har_source) - - for layer in amp: - xs += layer(x) - - x = xs / self.num_kernels - - return self.conv_post(self.act_post(x)).tanh() - - def remove_weight_norm(self): - if hasattr(self.conv_pre, "parametrizations") and "weight" in self.conv_pre.parametrizations: parametrize.remove_parametrizations(self.conv_pre, "weight", leave_parametrized=True) - else: remove_weight_norm(self.conv_pre) - - for up in self.upsamples: - if hasattr(up, "parametrizations") and "weight" in up.parametrizations: parametrize.remove_parametrizations(up, "weight", leave_parametrized=True) - else: remove_weight_norm(up) - - for amp in self.amps: - amp.remove_weight_norm() - - if hasattr(self.conv_post, "parametrizations") and "weight" in self.conv_post.parametrizations: parametrize.remove_parametrizations(self.conv_post, "weight", leave_parametrized=True) - else: remove_weight_norm(self.conv_post) \ No newline at end of file diff --git a/main/library/generators/hifigan.py b/main/library/generators/hifigan.py deleted file mode 100644 index 491e1955f0eadc5499d62f184debf148ea64a1e9..0000000000000000000000000000000000000000 --- a/main/library/generators/hifigan.py +++ /dev/null @@ -1,75 +0,0 @@ -import os -import sys -import torch - -import torch.nn.utils.parametrize as parametrize - -from torch.nn.utils import remove_weight_norm -from torch.nn.utils.parametrizations import weight_norm - -sys.path.append(os.getcwd()) - -from main.library.algorithm.commons import init_weights -from main.library.algorithm.residuals import ResBlock, LRELU_SLOPE - -class HiFiGANGenerator(torch.nn.Module): - def __init__( - self, - initial_channel, - resblock_kernel_sizes, - resblock_dilation_sizes, - upsample_rates, - upsample_initial_channel, - upsample_kernel_sizes, - gin_channels=0 - ): - super(HiFiGANGenerator, self).__init__() - self.num_kernels = len(resblock_kernel_sizes) - self.num_upsamples = len(upsample_rates) - self.conv_pre = torch.nn.Conv1d(initial_channel, upsample_initial_channel, 7, 1, padding=3) - self.ups = torch.nn.ModuleList() - self.resblocks = torch.nn.ModuleList() - - for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)): - self.ups.append( - weight_norm( - torch.nn.ConvTranspose1d( - upsample_initial_channel // (2**i), - upsample_initial_channel // (2 ** (i + 1)), - k, - u, - padding=(k - u) // 2 - ) - ) - ) - - ch = upsample_initial_channel // (2 ** (i + 1)) - for k, d in zip(resblock_kernel_sizes, resblock_dilation_sizes): - self.resblocks.append(ResBlock(ch, k, d)) - - self.conv_post = torch.nn.Conv1d(ch, 1, 7, 1, padding=3, bias=False) - self.ups.apply(init_weights) - if gin_channels != 0: self.cond = torch.nn.Conv1d(gin_channels, upsample_initial_channel, 1) - - def forward(self, x, g = None): - x = self.conv_pre(x) - if g is not None: x += self.cond(g) - - for i in range(self.num_upsamples): - x = self.ups[i](torch.nn.functional.leaky_relu(x, LRELU_SLOPE)) - xs = None - - for j in range(self.num_kernels): - if xs is None: xs = self.resblocks[i * self.num_kernels + j](x) - else: xs += self.resblocks[i * self.num_kernels + j](x) - x = xs / self.num_kernels - - return self.conv_post(torch.nn.functional.leaky_relu(x)).tanh() - - def remove_weight_norm(self): - for l in self.ups: - if hasattr(l, "parametrizations") and "weight" in l.parametrizations: parametrize.remove_parametrizations(l, "weight", leave_parametrized=True) - else: remove_weight_norm(l) - - for l in self.resblocks: - l.remove_weight_norm() \ No newline at end of file diff --git a/main/library/generators/mrf_hifigan.py b/main/library/generators/mrf_hifigan.py deleted file mode 100644 index 417a53a23a57454b3ca9e005a959310d502a6c7c..0000000000000000000000000000000000000000 --- a/main/library/generators/mrf_hifigan.py +++ /dev/null @@ -1,242 +0,0 @@ -import math -import torch - -import numpy as np -import torch.nn as nn -import torch.nn.functional as F -import torch.nn.utils.parametrize as parametrize - -from torch.nn.utils import remove_weight_norm -from torch.utils.checkpoint import checkpoint -from torch.nn.utils.parametrizations import weight_norm - -LRELU_SLOPE = 0.1 - -class MRFLayer(nn.Module): - def __init__( - self, - channels, - kernel_size, - dilation - ): - super().__init__() - self.conv1 = weight_norm( - nn.Conv1d( - channels, - channels, - kernel_size, - padding=(kernel_size * dilation - dilation) // 2, - dilation=dilation - ) - ) - self.conv2 = weight_norm( - nn.Conv1d( - channels, - channels, - kernel_size, - padding=kernel_size // 2, - dilation=1 - ) - ) - - def forward(self, x): - return x + self.conv2(F.leaky_relu(self.conv1(F.leaky_relu(x, LRELU_SLOPE)), LRELU_SLOPE)) - - def remove_weight_norm(self): - if hasattr(self.conv1, "parametrizations") and "weight" in self.conv1.parametrizations: parametrize.remove_parametrizations(self.conv1, "weight", leave_parametrized=True) - else: remove_weight_norm(self.conv1) - - if hasattr(self.conv2, "parametrizations") and "weight" in self.conv2.parametrizations: parametrize.remove_parametrizations(self.conv2, "weight", leave_parametrized=True) - else: remove_weight_norm(self.conv2) - -class MRFBlock(nn.Module): - def __init__(self, channels, kernel_size, dilations): - super().__init__() - self.layers = nn.ModuleList() - - for dilation in dilations: - self.layers.append(MRFLayer(channels, kernel_size, dilation)) - - def forward(self, x): - for layer in self.layers: - x = layer(x) - - return x - - def remove_weight_norm(self): - for layer in self.layers: - layer.remove_weight_norm() - -class SineGen(nn.Module): - def __init__( - self, - sampling_rate, - harmonic_num = 0, - sine_amp = 0.1, - noise_std = 0.003, - voiced_threshold = 0 - ): - super(SineGen, self).__init__() - self.sine_amp = sine_amp - self.noise_std = noise_std - self.harmonic_num = harmonic_num - self.dim = self.harmonic_num + 1 - self.sampling_rate = sampling_rate - self.voiced_threshold = voiced_threshold - - def _f02uv(self, f0): - return (f0 > self.voiced_threshold).float() - - def _f02sine(self, f0_values): - rad_values = (f0_values / self.sampling_rate) % 1 - rand_ini = torch.rand(f0_values.shape[0], f0_values.shape[2], dtype=f0_values.dtype, device=f0_values.device) - rand_ini[:, 0] = 0 - rad_values[:, 0, :] = rad_values[:, 0, :] + rand_ini - tmp_over_one = torch.cumsum(rad_values, 1) % 1 - tmp_over_one_idx = (tmp_over_one[:, 1:, :] - tmp_over_one[:, :-1, :]) < 0 - cumsum_shift = torch.zeros_like(rad_values) - cumsum_shift[:, 1:, :] = tmp_over_one_idx * -1.0 - - return (torch.cumsum(rad_values + cumsum_shift, dim=1) * 2 * np.pi).sin() - - def forward(self, f0): - with torch.no_grad(): - f0_buf = torch.zeros(f0.shape[0], f0.shape[1], self.dim, dtype=f0.dtype, device=f0.device) - f0_buf[:, :, 0] = f0[:, :, 0] - - for idx in np.arange(self.harmonic_num): - f0_buf[:, :, idx + 1] = f0_buf[:, :, 0] * (idx + 2) - - sine_waves = self._f02sine(f0_buf) * self.sine_amp - uv = self._f02uv(f0) - sine_waves = sine_waves * uv + ((uv * self.noise_std + (1 - uv) * self.sine_amp / 3) * torch.randn_like(sine_waves)) - - return sine_waves - -class SourceModuleHnNSF(nn.Module): - def __init__( - self, - sampling_rate, - harmonic_num = 0, - sine_amp = 0.1, - add_noise_std = 0.003, - voiced_threshold = 0 - ): - super(SourceModuleHnNSF, self).__init__() - self.sine_amp = sine_amp - self.noise_std = add_noise_std - self.l_sin_gen = SineGen(sampling_rate, harmonic_num, sine_amp, add_noise_std, voiced_threshold) - self.l_linear = nn.Linear(harmonic_num + 1, 1) - self.l_tanh = nn.Tanh() - - def forward(self, x): - return self.l_tanh( - self.l_linear( - self.l_sin_gen(x).to(dtype=self.l_linear.weight.dtype) - ) - ) - -class HiFiGANMRFGenerator(nn.Module): - def __init__( - self, - in_channel, - upsample_initial_channel, - upsample_rates, - upsample_kernel_sizes, - resblock_kernel_sizes, - resblock_dilations, - gin_channels, - sample_rate, - harmonic_num, - checkpointing = False - ): - super().__init__() - self.num_kernels = len(resblock_kernel_sizes) - self.checkpointing = checkpointing - self.f0_upsample = nn.Upsample(scale_factor=np.prod(upsample_rates)) - self.m_source = SourceModuleHnNSF(sample_rate, harmonic_num) - self.conv_pre = weight_norm(nn.Conv1d(in_channel, upsample_initial_channel, kernel_size=7, stride=1, padding=3)) - self.upsamples = nn.ModuleList() - self.noise_convs = nn.ModuleList() - stride_f0s = [ - math.prod(upsample_rates[i + 1 :]) if i + 1 < len(upsample_rates) else 1 - for i in range(len(upsample_rates)) - ] - - for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)): - self.upsamples.append( - weight_norm( - nn.ConvTranspose1d( - upsample_initial_channel // (2**i), - upsample_initial_channel // (2 ** (i + 1)), - kernel_size=k, - stride=u, - padding=((k - u) // 2) if u % 2 == 0 else (u // 2 + u % 2), - output_padding=u % 2 - ) - ) - ) - - stride = stride_f0s[i] - kernel = 1 if stride == 1 else stride * 2 - stride % 2 - - self.noise_convs.append( - nn.Conv1d( - 1, - upsample_initial_channel // (2 ** (i + 1)), - kernel_size=kernel, - stride=stride, - padding=0 if stride == 1 else (kernel - stride) // 2 - ) - ) - - self.mrfs = nn.ModuleList() - for i in range(len(self.upsamples)): - channel = upsample_initial_channel // (2 ** (i + 1)) - self.mrfs.append( - nn.ModuleList([ - MRFBlock( - channel, - kernel_size=k, - dilations=d - ) - for k, d in zip(resblock_kernel_sizes, resblock_dilations) - ]) - ) - - self.conv_post = weight_norm(nn.Conv1d(channel, 1, kernel_size=7, stride=1, padding=3)) - if gin_channels != 0: self.cond = nn.Conv1d(gin_channels, upsample_initial_channel, 1) - - def forward(self, x, f0, g = None): - har_source = self.m_source(self.f0_upsample(f0[:, None, :]).transpose(-1, -2)).transpose(-1, -2) - x = self.conv_pre(x) - if g is not None: x += self.cond(g) - - for ups, mrf, noise_conv in zip(self.upsamples, self.mrfs, self.noise_convs): - x = F.leaky_relu(x, LRELU_SLOPE) - - if self.training and self.checkpointing: - x = checkpoint(ups, x, use_reentrant=False) + noise_conv(har_source) - xs = sum([checkpoint(layer, x, use_reentrant=False) for layer in mrf]) - else: - x = ups(x) + noise_conv(har_source) - xs = sum([layer(x) for layer in mrf]) - - x = xs / self.num_kernels - - return self.conv_post(F.leaky_relu(x)).tanh() - - def remove_weight_norm(self): - if hasattr(self.conv_pre, "parametrizations") and "weight" in self.conv_pre.parametrizations: parametrize.remove_parametrizations(self.conv_pre, "weight", leave_parametrized=True) - else: remove_weight_norm(self.conv_pre) - - for up in self.upsamples: - if hasattr(up, "parametrizations") and "weight" in up.parametrizations: parametrize.remove_parametrizations(up, "weight", leave_parametrized=True) - else: remove_weight_norm(up) - - for mrf in self.mrfs: - for block in mrf: - block.remove_weight_norm() - - if hasattr(self.conv_post, "parametrizations") and "weight" in self.conv_post.parametrizations: parametrize.remove_parametrizations(self.conv_post, "weight", leave_parametrized=True) - else: remove_weight_norm(self.conv_post) \ No newline at end of file diff --git a/main/library/generators/nsf_hifigan.py b/main/library/generators/nsf_hifigan.py deleted file mode 100644 index f171ad9ada03129afedee5255e2a987b09280813..0000000000000000000000000000000000000000 --- a/main/library/generators/nsf_hifigan.py +++ /dev/null @@ -1,242 +0,0 @@ -import os -import sys -import math -import torch - -import numpy as np -import torch.nn.functional as F -import torch.nn.utils.parametrize as parametrize - -from torch.nn.utils import remove_weight_norm -from torch.utils.checkpoint import checkpoint -from torch.nn.utils.parametrizations import weight_norm - -sys.path.append(os.getcwd()) - -from main.library.algorithm.commons import init_weights -from main.library.algorithm.residuals import ResBlock, LRELU_SLOPE - -class SineGen2(torch.nn.Module): - def __init__( - self, - sampling_rate, - harmonic_num=0, - sine_amp=0.1, - noise_std=0.003, - voiced_threshold=0, - ): - super(SineGen2, self).__init__() - self.sine_amp = sine_amp - self.noise_std = noise_std - self.harmonic_num = harmonic_num - self.dim = self.harmonic_num + 1 - self.sampling_rate = sampling_rate - self.voiced_threshold = voiced_threshold - - def _f02uv(self, f0): - return (f0 > self.voiced_threshold).float() - - def _f02sine(self, f0_values): - rad_values = (f0_values / self.sampling_rate) % 1 - rand_ini = torch.rand(f0_values.shape[0], f0_values.shape[2], dtype=f0_values.dtype, device=f0_values.device) - rand_ini[:, 0] = 0 - rad_values[:, 0, :] = rad_values[:, 0, :] + rand_ini - tmp_over_one = torch.cumsum(rad_values, 1) % 1 - tmp_over_one_idx = (tmp_over_one[:, 1:, :] - tmp_over_one[:, :-1, :]) < 0 - cumsum_shift = torch.zeros_like(rad_values) - cumsum_shift[:, 1:, :] = tmp_over_one_idx * -1.0 - - return (torch.cumsum(rad_values + cumsum_shift, dim=1) * 2 * np.pi).sin() - - def forward(self, f0, upp): - with torch.no_grad(): - sine_waves = self._f02sine( - torch.multiply( - f0, - torch.FloatTensor([ - [range(1, self.harmonic_num + 2)] - ]).to(f0.device) - ) - ) * self.sine_amp - - uv = self._f02uv(f0) - sine_waves = sine_waves * uv + ((uv * self.noise_std + (1 - uv) * self.sine_amp / 3) * torch.randn_like(sine_waves)) - - return sine_waves - -class SineGen(torch.nn.Module): - def __init__( - self, - sampling_rate, - harmonic_num=0, - sine_amp=0.1, - noise_std=0.003, - voiced_threshold=0 - ): - super(SineGen, self).__init__() - self.sine_amp = sine_amp - self.noise_std = noise_std - self.harmonic_num = harmonic_num - self.dim = self.harmonic_num + 1 - self.sampling_rate = sampling_rate - self.voiced_threshold = voiced_threshold - - def _f02uv(self, f0): - return (f0 > self.voiced_threshold).float() - - def _f02sine(self, f0, upp): - rad = f0 / self.sampling_rate * torch.arange(1, upp + 1, dtype=f0.dtype, device=f0.device) - rad += F.pad((torch.fmod(rad[:, :-1, -1:].float() + 0.5, 1.0) - 0.5).cumsum(dim=1).fmod(1.0).to(f0), (0, 0, 1, 0), mode='constant') - rad = rad.reshape(f0.shape[0], -1, 1) - rad *= torch.arange(1, self.dim + 1, dtype=f0.dtype, device=f0.device).reshape(1, 1, -1) - rand_ini = torch.rand(1, 1, self.dim, device=f0.device) - rand_ini[..., 0] = 0 - rad += rand_ini - - return (2 * np.pi * rad).sin() - - def forward(self, f0, upp): - with torch.no_grad(): - f0 = f0.unsqueeze(-1) - sine_waves = self._f02sine(f0, upp) * self.sine_amp - uv = F.interpolate(self._f02uv(f0).transpose(2, 1), scale_factor=float(upp), mode="nearest").transpose(2, 1) - sine_waves = sine_waves * uv + ((uv * self.noise_std + (1 - uv) * self.sine_amp / 3) * torch.randn_like(sine_waves)) - - return sine_waves - -class SourceModuleHnNSF(torch.nn.Module): - def __init__( - self, - sample_rate, - harmonic_num=0, - sine_amp=0.1, - add_noise_std=0.003, - voiced_threshod=0, - sinegen_version="v1" - ): - super(SourceModuleHnNSF, self).__init__() - self.sine_amp = sine_amp - self.noise_std = add_noise_std - sine_gen_fn = SineGen if sinegen_version == "v1" else SineGen2 - self.l_sin_gen = sine_gen_fn(sample_rate, harmonic_num, sine_amp, add_noise_std, voiced_threshod) - self.l_linear = torch.nn.Linear(harmonic_num + 1, 1) - self.l_tanh = torch.nn.Tanh() - - def forward(self, x, upsample_factor = 1): - return self.l_tanh( - self.l_linear( - self.l_sin_gen( - x, - upsample_factor - ).to(dtype=self.l_linear.weight.dtype) - ) - ) - -class HiFiGANNSFGenerator(torch.nn.Module): - def __init__( - self, - initial_channel, - resblock_kernel_sizes, - resblock_dilation_sizes, - upsample_rates, - upsample_initial_channel, - upsample_kernel_sizes, - gin_channels, - sr, - checkpointing = False, - harmonic_num = 0 - ): - super(HiFiGANNSFGenerator, self).__init__() - self.num_kernels = len(resblock_kernel_sizes) - self.num_upsamples = len(upsample_rates) - self.upp = math.prod(upsample_rates) - self.harmonic_num = harmonic_num - self.f0_upsamp = torch.nn.Upsample(scale_factor=self.upp) - self.m_source = SourceModuleHnNSF(sample_rate=sr, harmonic_num=harmonic_num, sinegen_version="v1" if harmonic_num == 0 else "v2") - - self.conv_pre = torch.nn.Conv1d(initial_channel, upsample_initial_channel, 7, 1, padding=3) - self.checkpointing = checkpointing - - self.ups = torch.nn.ModuleList() - self.noise_convs = torch.nn.ModuleList() - - channels = [ - upsample_initial_channel // (2 ** (i + 1)) - for i in range(self.num_upsamples) - ] - stride_f0s = [ - math.prod(upsample_rates[i + 1 :]) if i + 1 < self.num_upsamples else 1 - for i in range(self.num_upsamples) - ] - - for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)): - self.ups.append( - weight_norm( - torch.nn.ConvTranspose1d( - upsample_initial_channel // (2**i), - channels[i], - k, - u, - padding=((k - u) // 2) if u % 2 == 0 else (u // 2 + u % 2), - output_padding=u % 2 - ) - ) - ) - - stride = stride_f0s[i] - kernel = 1 if stride == 1 else stride * 2 - stride % 2 - - self.noise_convs.append( - torch.nn.Conv1d( - 1, - channels[i], - kernel_size=kernel, - stride=stride, - padding=0 if stride == 1 else (kernel - stride) // 2 - ) - ) - - self.resblocks = torch.nn.ModuleList([ - ResBlock(channels[i], k, d) - for i in range(len(self.ups)) - for k, d in zip(resblock_kernel_sizes, resblock_dilation_sizes) - ]) - self.conv_post = torch.nn.Conv1d(channels[-1], 1, 7, 1, padding=3, bias=bool(harmonic_num != 0)) - if harmonic_num != 0: self.conv_pre, self.conv_post = weight_norm(self.conv_pre), weight_norm(self.conv_post) - - self.ups.apply(init_weights) - if gin_channels != 0: self.cond = torch.nn.Conv1d(gin_channels, upsample_initial_channel, 1) - - def forward(self, x, f0, g = None): - if self.harmonic_num != 0: f0 = self.f0_upsamp(f0[:, None]).transpose(1, 2) - har_source = self.m_source(f0, self.upp).transpose(1, 2) - x = self.conv_pre(x) - if g is not None: x += self.cond(g) - - for i, (ups, noise_convs) in enumerate(zip(self.ups, self.noise_convs)): - x = F.leaky_relu(x, LRELU_SLOPE) - - if self.training and self.checkpointing: - x = checkpoint(ups, x, use_reentrant=False) + noise_convs(har_source) - xs = sum([checkpoint(resblock, x, use_reentrant=False) for j, resblock in enumerate(self.resblocks) if j in range(i * self.num_kernels, (i + 1) * self.num_kernels)]) - else: - x = ups(x) + noise_convs(har_source) - xs = sum([resblock(x) for j, resblock in enumerate(self.resblocks) if j in range(i * self.num_kernels, (i + 1) * self.num_kernels)]) - - x = xs / self.num_kernels - - return self.conv_post(F.leaky_relu(x)).tanh() - - def remove_weight_norm(self): - for l in self.ups: - if hasattr(l, "parametrizations") and "weight" in l.parametrizations: parametrize.remove_parametrizations(l, "weight", leave_parametrized=True) - else: remove_weight_norm(l) - - if self.harmonic_num != 0: - if hasattr(self.conv_pre, "parametrizations") and "weight" in self.conv_pre.parametrizations: parametrize.remove_parametrizations(self.conv_pre, "weight", leave_parametrized=True) - else: remove_weight_norm(self.conv_pre) - if hasattr(self.conv_post, "parametrizations") and "weight" in self.conv_post.parametrizations: parametrize.remove_parametrizations(self.conv_post, "weight", leave_parametrized=True) - else: remove_weight_norm(self.conv_post) - - for l in self.resblocks: - l.remove_weight_norm() \ No newline at end of file diff --git a/main/library/generators/refinegan.py b/main/library/generators/refinegan.py deleted file mode 100644 index 1898f90b8f7548d116ede56d66d8f1c1e090929e..0000000000000000000000000000000000000000 --- a/main/library/generators/refinegan.py +++ /dev/null @@ -1,282 +0,0 @@ -import os -import sys -import torch -import torchaudio - -import numpy as np -import torch.nn as nn -import torch.nn.functional as F -import torch.nn.utils.parametrize as parametrize - -from torch.utils.checkpoint import checkpoint -from torch.nn.utils import remove_weight_norm -from torch.nn.utils.parametrizations import weight_norm - -sys.path.append(os.getcwd()) - -from main.library.algorithm.commons import init_weights, get_padding - -class ResBlock(nn.Module): - def __init__(self, channels, kernel_size = 7, dilation = (1, 3, 5), leaky_relu_slope = 0.2): - super().__init__() - self.leaky_relu_slope = leaky_relu_slope - self.convs1 = nn.ModuleList([ - weight_norm( - nn.Conv1d( - channels, - channels, - kernel_size, - stride=1, - dilation=d, - padding=get_padding(kernel_size, d) - ) - ) - for d in dilation - ]) - self.convs1.apply(init_weights) - - self.convs2 = nn.ModuleList([ - weight_norm( - nn.Conv1d( - channels, - channels, - kernel_size, - stride=1, - dilation=1, - padding=get_padding(kernel_size, 1) - ) - ) - for _ in dilation - ]) - self.convs2.apply(init_weights) - - def forward(self, x): - for c1, c2 in zip(self.convs1, self.convs2): - x = c2(F.leaky_relu(c1(F.leaky_relu(x, self.leaky_relu_slope)), self.leaky_relu_slope)) + x - - return x - - def remove_weight_norm(self): - for c1, c2 in zip(self.convs1, self.convs2): - if hasattr(c1, "parametrizations") and "weight" in c1.parametrizations: parametrize.remove_parametrizations(c1, "weight", leave_parametrized=True) - else: remove_weight_norm(c1) - - if hasattr(c2, "parametrizations") and "weight" in c2.parametrizations: parametrize.remove_parametrizations(c2, "weight", leave_parametrized=True) - else: remove_weight_norm(c2) - -class AdaIN(nn.Module): - def __init__( - self, - *, - channels, - leaky_relu_slope = 0.2 - ): - super().__init__() - self.weight = nn.Parameter(torch.ones(channels) * 1e-4) - self.activation = nn.LeakyReLU(leaky_relu_slope) - - def forward(self, x): - return self.activation(x + (torch.randn_like(x) * self.weight[None, :, None])) - -class ParallelResBlock(nn.Module): - def __init__( - self, - *, - in_channels, - out_channels, - kernel_sizes = (3, 7, 11), - dilation = (1, 3, 5), - leaky_relu_slope = 0.2 - ): - super().__init__() - self.in_channels = in_channels - self.out_channels = out_channels - self.input_conv = nn.Conv1d(in_channels=in_channels, out_channels=out_channels, kernel_size=7, stride=1, padding=3) - self.input_conv.apply(init_weights) - self.blocks = nn.ModuleList([ - nn.Sequential( - AdaIN( - channels=out_channels - ), - ResBlock( - out_channels, - kernel_size=kernel_size, - dilation=dilation, - leaky_relu_slope=leaky_relu_slope - ), - AdaIN( - channels=out_channels - ) - ) - for kernel_size in kernel_sizes - ]) - - def forward(self, x): - x = self.input_conv(x) - return torch.stack([block(x) for block in self.blocks], dim=0).mean(dim=0) - - def remove_weight_norm(self): - for block in self.blocks: - block[1].remove_weight_norm() - -class SineGen(nn.Module): - def __init__( - self, - sampling_rate, - harmonic_num=0, - sine_amp=0.1, - noise_std=0.003, - voiced_threshold=0 - ): - super(SineGen, self).__init__() - self.sine_amp = sine_amp - self.noise_std = noise_std - self.harmonic_num = harmonic_num - self.dim = self.harmonic_num + 1 - self.sampling_rate = sampling_rate - self.voiced_threshold = voiced_threshold - self.merge = nn.Sequential(nn.Linear(self.dim, 1, bias=False), nn.Tanh()) - - def _f02uv(self, f0): - return (f0 > self.voiced_threshold).float() - - def _f02sine(self, f0_values): - rad_values = (f0_values / self.sampling_rate) % 1 - rand_ini = torch.rand(f0_values.shape[0], f0_values.shape[2], dtype=f0_values.dtype, device=f0_values.device) - - rand_ini[:, 0] = 0 - rad_values[:, 0, :] = rad_values[:, 0, :] + rand_ini - - tmp_over_one = torch.cumsum(rad_values, 1) % 1 - tmp_over_one_idx = (tmp_over_one[:, 1:, :] - tmp_over_one[:, :-1, :]) < 0 - - cumsum_shift = torch.zeros_like(rad_values) - cumsum_shift[:, 1:, :] = tmp_over_one_idx * -1.0 - - return (torch.cumsum(rad_values + cumsum_shift, dim=1) * 2 * np.pi).sin() - - def forward(self, f0): - with torch.no_grad(): - f0_buf = torch.zeros(f0.shape[0], f0.shape[1], self.dim, dtype=f0.dtype, device=f0.device) - f0_buf[:, :, 0] = f0[:, :, 0] - - for idx in np.arange(self.harmonic_num): - f0_buf[:, :, idx + 1] = f0_buf[:, :, 0] * (idx + 2) - - sine_waves = self._f02sine(f0_buf) * self.sine_amp - uv = self._f02uv(f0) - sine_waves = sine_waves * uv + ((uv * self.noise_std + (1 - uv) * self.sine_amp / 3) * torch.randn_like(sine_waves)) - - return self.merge(sine_waves) - -class RefineGANGenerator(nn.Module): - def __init__( - self, - *, - sample_rate = 44100, - upsample_rates = (8, 8, 2, 2), - leaky_relu_slope = 0.2, - num_mels = 128, - start_channels = 16, - gin_channels = 256, - checkpointing = False, - upsample_initial_channel = 512 - ): - super().__init__() - self.upsample_rates = upsample_rates - self.checkpointing = checkpointing - self.leaky_relu_slope = leaky_relu_slope - self.upp = np.prod(upsample_rates) - self.m_source = SineGen(sample_rate) - self.pre_conv = weight_norm(nn.Conv1d(1, 16, 7, 1, padding=3)) - channels = start_channels - size = self.upp - self.downsample_blocks = nn.ModuleList([]) - self.df0 = [] - - for i, _ in enumerate(upsample_rates): - new_size = int(size / upsample_rates[-i - 1]) - self.df0.append([size, new_size]) - size = new_size - - new_channels = channels * 2 - self.downsample_blocks.append(weight_norm(nn.Conv1d(channels, new_channels, 7, 1, padding=3))) - channels = new_channels - - channels = upsample_initial_channel - self.mel_conv = weight_norm(nn.Conv1d(num_mels, channels // 2, 7, 1, padding=3)) - self.mel_conv.apply(init_weights) - - if gin_channels != 0: self.cond = nn.Conv1d(256, channels // 2, 1) - - self.upsample_blocks = nn.ModuleList([]) - self.upsample_conv_blocks = nn.ModuleList([]) - - for rate in upsample_rates: - new_channels = channels // 2 - self.upsample_blocks.append( - nn.Upsample( - scale_factor=rate, - mode="linear" - ) - ) - self.upsample_conv_blocks.append( - ParallelResBlock( - in_channels=channels + channels // 4, - out_channels=new_channels, - kernel_sizes=(3, 7, 11), - dilation=(1, 3, 5), - leaky_relu_slope=leaky_relu_slope - ) - ) - channels = new_channels - - self.conv_post = weight_norm(nn.Conv1d(channels, 1, 7, 1, padding=3, bias=False)) - self.conv_post.apply(init_weights) - - def forward(self, mel, f0, g = None): - f0_size = mel.shape[-1] - har_source = self.m_source(F.interpolate(f0.unsqueeze(1), size=f0_size * self.upp, mode="linear").transpose(1, 2)).transpose(1, 2) - x = self.pre_conv(har_source) - downs = [] - - for block, (old_size, new_size) in zip(self.downsample_blocks, self.df0): - x = F.leaky_relu(x, self.leaky_relu_slope) - downs.append(x) - - x = torchaudio.functional.resample( - x.contiguous(), - orig_freq=int(f0_size * old_size), - new_freq=int(f0_size * new_size), - lowpass_filter_width=64, - rolloff=0.9475937167399596, - resampling_method="sinc_interp_kaiser", - beta=14.769656459379492 - ) - x = block(x) - - mel = self.mel_conv(mel) - if g is not None: mel += self.cond(g) - - x = torch.cat([mel, x], dim=1) - for ups, res, down in zip(self.upsample_blocks, self.upsample_conv_blocks, reversed(downs)): - x = F.leaky_relu(x, self.leaky_relu_slope) - x = checkpoint(res, torch.cat([checkpoint(ups, x, use_reentrant=False), down], dim=1), use_reentrant=False) if self.training and self.checkpointing else res(torch.cat([ups(x), down], dim=1)) - - return self.conv_post(F.leaky_relu(x, self.leaky_relu_slope)).tanh() - - def remove_weight_norm(self): - if hasattr(self.pre_conv, "parametrizations") and "weight" in self.pre_conv.parametrizations: parametrize.remove_parametrizations(self.pre_conv, "weight", leave_parametrized=True) - else: remove_weight_norm(self.pre_conv) - - if hasattr(self.mel_conv, "parametrizations") and "weight" in self.mel_conv.parametrizations: parametrize.remove_parametrizations(self.mel_conv, "weight", leave_parametrized=True) - else: remove_weight_norm(self.mel_conv) - - if hasattr(self.conv_post, "parametrizations") and "weight" in self.conv_post.parametrizations: parametrize.remove_parametrizations(self.conv_post, "weight", leave_parametrized=True) - else: remove_weight_norm(self.conv_post) - - for block in self.downsample_blocks: - block.remove_weight_norm() - - for block in self.upsample_conv_blocks: - block.remove_weight_norm() \ No newline at end of file diff --git a/main/library/onnx/onnx_export.py b/main/library/onnx/onnx_export.py deleted file mode 100644 index ed8b232f60d4b764a8431daca3aa4bda0db4225d..0000000000000000000000000000000000000000 --- a/main/library/onnx/onnx_export.py +++ /dev/null @@ -1,174 +0,0 @@ -import os -import io -import sys -import onnx -import json -import torch -import onnxslim -import warnings - -sys.path.append(os.getcwd()) - -from main.app.variables import logger, config -from main.library.algorithm.synthesizers import Synthesizer, SynthesizerSVC - -if not config.debug_mode: warnings.filterwarnings("ignore") - -FEATS_LENGTH = 200 - -def autocast(model): - orig_forward = model.forward - - def _forward(*args, **kwargs): - dtype = next(model.parameters()).dtype - return orig_forward(*[a.to(dtype) if isinstance(a, torch.Tensor) and a.dtype.is_floating_point else a for a in args], **kwargs) - - model.forward = _forward - return model - -def onnx_exporter(input_path, output_path, is_half=False, device="cpu"): - if not device.startswith("cuda") or config.is_zluda: device = "cpu" - - cpt = (torch.load(input_path, map_location="cpu", weights_only=True) if os.path.isfile(input_path) else None) - cpt["config"][-3] = cpt["weight"]["emb_g.weight"].shape[0] - - ( - model_name, - model_author, - epochs, steps, - version, - f0, - model_hash, - vocoder, - creation_date, - energy_use, - speakers_id, - architecture - ) = ( - cpt.get("model_name", None), - cpt.get("author", None), - cpt.get("epoch", None), - cpt.get("step", None), - cpt.get("version", "v1"), - cpt.get("f0", 1), - cpt.get("model_hash", None), - cpt.get("vocoder", "Default"), - cpt.get("creation_date", None), - cpt.get("energy", False), - cpt.get("speakers_id", 1), - cpt.get("architecture", "RVC") - ) - - text_enc_hidden_dim = 768 if version == "v2" else 256 - tgt_sr = cpt["config"][-1] - - if architecture == "RVC": - net_g = Synthesizer( - *cpt["config"], - use_f0=f0, - text_enc_hidden_dim=text_enc_hidden_dim, - vocoder=vocoder, - checkpointing=False, - energy=energy_use, - onnx=True - ) - else: - net_g = SynthesizerSVC( - *cpt["config"], - text_enc_hidden_dim=text_enc_hidden_dim, - vocoder=vocoder, - checkpointing=False, - noise_scale=0.4, - onnx=True - ) - - net_g.forward = net_g.onnx_infer - net_g.load_state_dict(cpt["weight"], strict=False) - net_g.eval().to(device).to(torch.float16 if is_half else torch.float32) - net_g.remove_weight_norm() - if is_half: net_g = autocast(net_g) - - phone = torch.rand(1, FEATS_LENGTH, text_enc_hidden_dim).to(device) - phone_length = torch.tensor([FEATS_LENGTH]).long().to(device) - sid = torch.LongTensor([0]).to(device) - - args = [phone, phone_length] - input_names = ["phone", "phone_lengths"] - dynamic_axes = { - "phone": {0: "B", 1: "T"}, - "phone_lengths": {0: "B"}, - } - - if f0: - pitch = torch.randint(size=(1, FEATS_LENGTH), low=5, high=255).to(device) - nsff0 = torch.rand(1, FEATS_LENGTH).to(device) - - args += [pitch, nsff0] - input_names += ["pitch", "nsff0"] - dynamic_axes.update({ - "pitch": {0: "B", 1: "T"}, - "nsff0": {0: "B", 1: "T"}, - }) - - args += [sid] - input_names += ["sid"] - dynamic_axes.update({ - "sid": {0: "B"} - }) - - if energy_use: - energy = torch.rand(1, FEATS_LENGTH).to(device) - args.append(energy) - - input_names.append("energy") - dynamic_axes.update({ - "energy": {0: "B", 1: "T"}, - }) - - try: - with io.BytesIO() as model: - torch.onnx.export( - net_g, - tuple(args), - model, - do_constant_folding=True, - opset_version=17, - verbose=False, - input_names=input_names, - output_names=["audio"], - dynamic_axes=dynamic_axes, - dynamo=False - ) - - model = onnxslim.slim(onnx.load_model_from_string(model.getvalue())) - model.metadata_props.append( - onnx.StringStringEntryProto( - key="model_info", - value=json.dumps( - { - "model_name": model_name, - "author": model_author, - "epoch": epochs, - "step": steps, - "version": version, - "sr": tgt_sr, - "f0": f0, - "model_hash": model_hash, - "creation_date": creation_date, - "vocoder": vocoder, - "text_enc_hidden_dim": text_enc_hidden_dim, - "energy": energy_use, - "speakers_id": speakers_id, - "architecture": architecture - } - ) - ) - ) - - onnx.save(model, output_path) - return output_path - except: - import traceback - logger.error(traceback.format_exc()) - - return None \ No newline at end of file diff --git a/main/library/onnx/wrapper.py b/main/library/onnx/wrapper.py deleted file mode 100644 index d2499d80d09b0a166d7f71c6295dc5ac3c28dcdf..0000000000000000000000000000000000000000 --- a/main/library/onnx/wrapper.py +++ /dev/null @@ -1,94 +0,0 @@ -import json -import torch -import onnxruntime - -import numpy as np - -class ONNXRVC: - def __init__( - self, - model_path, - providers, - log_severity_level = 3 - ): - sess_options = onnxruntime.SessionOptions() - sess_options.log_severity_level = log_severity_level - - self.net_g = onnxruntime.InferenceSession( - model_path, - sess_options=sess_options, - providers=providers - ) - - metadata_dict = json.loads(self.net_g.get_modelmeta().custom_metadata_map["model_info"]) - - self.cpt = {} - self.cpt["tgt_sr"] = metadata_dict.get("sr", 32000) - self.cpt["use_f0"] = metadata_dict.get("f0", 1) - self.cpt["version"] = metadata_dict.get("version", "v1") - self.cpt["energy"] = metadata_dict.get("energy", False) - - def get_onnx_argument( - self, - feats, - p_len, - sid, - pitch, - pitchf, - energy - ): - inputs = { - self.net_g.get_inputs()[0].name: feats.cpu().numpy().astype(np.float32), - self.net_g.get_inputs()[1].name: p_len.cpu().numpy() - } - - if self.cpt["energy"]: - if self.cpt["use_f0"]: - inputs.update({ - self.net_g.get_inputs()[2].name: pitch.cpu().numpy().astype(np.int64), - self.net_g.get_inputs()[3].name: pitchf.cpu().numpy().astype(np.float32), - self.net_g.get_inputs()[4].name: np.array([sid.cpu().item()], dtype=np.int64), - self.net_g.get_inputs()[5].name: energy.cpu().numpy().astype(np.float32) - }) - else: - inputs.update({ - self.net_g.get_inputs()[2].name: np.array([sid.cpu().item()], dtype=np.int64), - self.net_g.get_inputs()[3].name: energy.cpu().numpy().astype(np.float32) - }) - else: - if self.cpt["use_f0"]: - inputs.update({ - self.net_g.get_inputs()[2].name: pitch.cpu().numpy().astype(np.int64), - self.net_g.get_inputs()[3].name: pitchf.cpu().numpy().astype(np.float32), - self.net_g.get_inputs()[4].name: np.array([sid.cpu().item()], dtype=np.int64) - }) - - return inputs - - def to(self, device = "cpu"): - self.device = device - return self - - def infer( - self, - feats = None, - p_len = None, - pitch = None, - pitchf = None, - sid = None, - energy = None - ): - output = self.net_g.run( - [self.net_g.get_outputs()[0].name], ( - self.get_onnx_argument( - feats, - p_len, - sid, - pitch, - pitchf, - energy, - ) - ) - ) - - return [torch.from_numpy(output[0]).to(self.device)] \ No newline at end of file diff --git a/main/library/optimizers/adabelief.py b/main/library/optimizers/adabelief.py deleted file mode 100644 index c271ce084eb0ef3ed6ad11645f63e8f9fadf896e..0000000000000000000000000000000000000000 --- a/main/library/optimizers/adabelief.py +++ /dev/null @@ -1,165 +0,0 @@ -import math -import torch - -from torch.optim.optimizer import Optimizer - -class AdaBelief(Optimizer): - def __init__( - self, - params, - lr = 1e-4, - betas = (0.8, 0.99), - eps = 1e-10, - weight_decay = 0.0, - foreach = True, - use_gc = False, - ): - if lr <= 0.0: - raise ValueError - if eps < 0.0: - raise ValueError - if not 0.0 <= betas[0] < 1.0: - raise ValueError - if not 0.0 <= betas[1] < 1.0: - raise ValueError - if weight_decay < 0: - raise ValueError - - defaults = dict( - lr=lr, - betas=betas, - eps=eps, - weight_decay=weight_decay, - foreach=foreach, - use_gc=use_gc, - ) - super().__init__(params, defaults) - - def __setstate__(self, state): - super().__setstate__(state) - for group in self.param_groups: - group.setdefault("foreach", True) - group.setdefault("use_gc", False) - - @torch.no_grad() - def step(self, closure=None): - loss = None - if closure is not None: - with torch.enable_grad(): - loss = closure() - - for group in self.param_groups: - if group["foreach"]: - self._step_foreach(group) - else: - self._step_single(group) - - return loss - - def _step_single(self, group): - beta1, beta2 = group["betas"] - eps = group["eps"] - lr = group["lr"] - weight_decay = group["weight_decay"] - use_gc = group["use_gc"] - - for p in group["params"]: - if p.grad is None: - continue - - grad = p.grad - if grad.is_sparse: - raise RuntimeError - - if use_gc and grad.dim() > 1: - grad.add_(-grad.mean(dim=tuple(range(1, grad.dim())), keepdim=True)) - - state = self.state[p] - - if len(state) == 0: - state["step"] = 0 - state["exp_avg"] = torch.zeros_like(p, memory_format=torch.preserve_format) - state["exp_avg_var"] = torch.zeros_like(p, memory_format=torch.preserve_format) - - exp_avg = state["exp_avg"] - exp_avg_var = state["exp_avg_var"] - - state["step"] += 1 - step = state["step"] - - bias_correction1 = 1 - beta1 ** step - bias_correction2 = 1 - beta2 ** step - - if weight_decay != 0: - p.mul_(1 - lr * weight_decay) - - exp_avg.mul_(beta1).add_(grad, alpha=1 - beta1) - - grad_residual = grad - exp_avg - exp_avg_var.mul_(beta2).addcmul_(grad_residual, grad_residual, value=1 - beta2) - - denom = (exp_avg_var.add(eps)).sqrt() / math.sqrt(bias_correction2) - - step_size = lr / bias_correction1 - p.addcdiv_(exp_avg, denom, value=-step_size) - - def _step_foreach(self, group): - beta1, beta2 = group["betas"] - eps = group["eps"] - lr = group["lr"] - weight_decay = group["weight_decay"] - use_gc = group["use_gc"] - - params_with_grad, grads, exp_avgs, exp_avg_vars = [], [], [], [] - - for p in group["params"]: - if p.grad is None: - continue - - params_with_grad.append(p) - grad = p.grad - - if use_gc and grad.dim() > 1: - grad.add_(-grad.mean(dim=tuple(range(1, grad.dim())), keepdim=True)) - - grads.append(grad) - - state = self.state[p] - - if len(state) == 0: - state["step"] = 0 - state["exp_avg"] = torch.zeros_like(p, memory_format=torch.preserve_format) - state["exp_avg_var"] = torch.zeros_like(p, memory_format=torch.preserve_format) - - exp_avgs.append(state["exp_avg"]) - exp_avg_vars.append(state["exp_avg_var"]) - - if not params_with_grad: - return - - state = self.state[params_with_grad[0]] - state["step"] += 1 - step = state["step"] - for p in params_with_grad[1:]: - self.state[p]["step"] = step - - bias_correction1 = 1 - beta1 ** step - bias_correction2 = 1 - beta2 ** step - - if weight_decay != 0: - torch._foreach_mul_(params_with_grad, 1 - lr * weight_decay) - - torch._foreach_mul_(exp_avgs, beta1) - torch._foreach_add_(exp_avgs, grads, alpha=1 - beta1) - - grad_residuals = torch._foreach_sub(grads, exp_avgs) - torch._foreach_mul_(exp_avg_vars, beta2) - torch._foreach_addcmul_(exp_avg_vars, grad_residuals, grad_residuals, value=1 - beta2) - - denom = torch._foreach_add(exp_avg_vars, eps) - denom = torch._foreach_sqrt(denom) - torch._foreach_div_(denom, math.sqrt(bias_correction2)) - - step_size = lr / bias_correction1 - updates = torch._foreach_div(exp_avgs, denom) - torch._foreach_add_(params_with_grad, updates, alpha=-step_size) \ No newline at end of file diff --git a/main/library/optimizers/adabeliefv2.py b/main/library/optimizers/adabeliefv2.py deleted file mode 100644 index a811b76c5be214ba1b2d312787ffa2970998bfcf..0000000000000000000000000000000000000000 --- a/main/library/optimizers/adabeliefv2.py +++ /dev/null @@ -1,177 +0,0 @@ -import math -import torch - -from torch.optim.optimizer import Optimizer - -class AdaBeliefV2(Optimizer): - def __init__( - self, - params, - lr = 1e-4, - betas = (0.9, 0.999), - eps = 1e-8, - weight_decay = 0, - amsgrad = True, - foreach = True, - ): - if lr <= 0.0: - raise ValueError - if eps < 0.0: - raise ValueError - if not 0.0 <= betas[0] < 1.0: - raise ValueError - if not 0.0 <= betas[1] < 1.0: - raise ValueError - if weight_decay < 0: - raise ValueError - - defaults = dict( - lr=lr, - betas=betas, - eps=eps, - weight_decay=weight_decay, - amsgrad=amsgrad, - foreach=foreach, - ) - super().__init__(params, defaults) - - def __setstate__(self, state): - super().__setstate__(state) - for group in self.param_groups: - group.setdefault("foreach", True) - group.setdefault("amsgrad", False) - - @torch.no_grad() - def step(self, closure=None): - loss = None - if closure is not None: - with torch.enable_grad(): - loss = closure() - - for group in self.param_groups: - if group["foreach"]: - self._step_foreach(group) - else: - self._step_single(group) - - return loss - - def _step_single(self, group): - beta1, beta2 = group["betas"] - eps = group["eps"] - lr = group["lr"] - weight_decay = group["weight_decay"] - amsgrad = group["amsgrad"] - - for p in group["params"]: - if p.grad is None: - continue - - grad = p.grad - if grad.is_sparse: - raise RuntimeError - - if weight_decay != 0: - p.mul_(1 - lr * weight_decay) - - state = self.state[p] - - if len(state) == 0: - state["step"] = 0 - state["exp_avg"] = torch.zeros_like(p, memory_format=torch.preserve_format) - state["exp_avg_var"] = torch.zeros_like(p, memory_format=torch.preserve_format) - if amsgrad: - state["max_exp_avg_var"] = torch.zeros_like(p, memory_format=torch.preserve_format) - - exp_avg = state["exp_avg"] - exp_avg_var = state["exp_avg_var"] - - state["step"] += 1 - step = state["step"] - - exp_avg.mul_(beta1).add_(grad, alpha=1 - beta1) - grad_residual = grad - exp_avg - exp_avg_var.mul_(beta2).addcmul_(grad_residual, grad_residual, value=1 - beta2) - - if amsgrad: - max_exp_avg_var = state["max_exp_avg_var"] - torch.maximum(max_exp_avg_var, exp_avg_var, out=max_exp_avg_var) - denom = max_exp_avg_var.add(eps).sqrt() - else: - denom = exp_avg_var.add(eps).sqrt() - - bias_correction1 = 1 - beta1 ** step - bias_correction2 = 1 - beta2 ** step - - step_size = lr * math.sqrt(bias_correction2) / bias_correction1 - - p.addcdiv_(exp_avg, denom, value=-step_size) - - def _step_foreach(self, group): - beta1, beta2 = group["betas"] - eps = group["eps"] - lr = group["lr"] - weight_decay = group["weight_decay"] - amsgrad = group["amsgrad"] - params_with_grad, grads, exp_avgs, exp_avg_vars, max_exp_avg_vars = [], [], [], [], [] - - for p in group["params"]: - if p.grad is None: - continue - params_with_grad.append(p) - grads.append(p.grad) - - state = self.state[p] - if len(state) == 0: - state["step"] = 0 - state["exp_avg"] = torch.zeros_like(p, memory_format=torch.preserve_format) - state["exp_avg_var"] = torch.zeros_like(p, memory_format=torch.preserve_format) - if amsgrad: - state["max_exp_avg_var"] = torch.zeros_like(p, memory_format=torch.preserve_format) - - exp_avgs.append(state["exp_avg"]) - exp_avg_vars.append(state["exp_avg_var"]) - if amsgrad: - max_exp_avg_vars.append(state["max_exp_avg_var"]) - - if not params_with_grad: - return - - state = self.state[params_with_grad[0]] - state["step"] += 1 - step = state["step"] - for p in params_with_grad[1:]: - self.state[p]["step"] = step - - if weight_decay != 0: - torch._foreach_mul_(params_with_grad, 1 - lr * weight_decay) - - torch._foreach_mul_(exp_avgs, beta1) - torch._foreach_add_(exp_avgs, grads, alpha=1 - beta1) - - grad_residuals = torch._foreach_sub(grads, exp_avgs) - torch._foreach_mul_(exp_avg_vars, beta2) - torch._foreach_addcmul_(exp_avg_vars, grad_residuals, grad_residuals, value=1 - beta2) - - if amsgrad: - torch._foreach_maximum_(max_exp_avg_vars, exp_avg_vars) - denom = torch._foreach_add(max_exp_avg_vars, eps) - else: denom = torch._foreach_add(exp_avg_vars, eps) - - torch._foreach_sqrt_(denom) - - bias_correction1 = 1 - beta1 ** step - bias_correction2 = 1 - beta2 ** step - step_size = lr * math.sqrt(bias_correction2) / bias_correction1 - - torch._foreach_addcdiv_(params_with_grad, exp_avgs, denom, value=-step_size) - - -def get_inverse_sqrt_scheduler(optimizer, warmup_epochs=15, last_epoch=-1): - def lr_lambda(current_step): - ep = current_step + 1 - - if ep < warmup_epochs: return float(ep) / float(max(1, warmup_epochs)) - return (warmup_epochs ** 0.5) / (ep ** 0.5) - - return torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=lr_lambda, last_epoch=last_epoch) \ No newline at end of file diff --git a/main/library/optimizers/anyprecision_optimizer.py b/main/library/optimizers/anyprecision_optimizer.py deleted file mode 100644 index be0ea1c336786cb076b28c55aecd1e52dbfc40fc..0000000000000000000000000000000000000000 --- a/main/library/optimizers/anyprecision_optimizer.py +++ /dev/null @@ -1,87 +0,0 @@ -import torch - -from torch.optim.optimizer import Optimizer - -class AnyPrecisionAdamW(Optimizer): - def __init__( - self, - params, - lr=1e-3, - betas=(0.9, 0.999), - eps=1e-8, - weight_decay=0.0, - use_kahan_summation=True, - momentum_dtype=torch.bfloat16, - variance_dtype=torch.bfloat16, - compensation_buffer_dtype=torch.bfloat16 - ): - defaults = dict( - lr=lr, - betas=betas, - eps=eps, - weight_decay=weight_decay, - use_kahan_summation=use_kahan_summation, - momentum_dtype=momentum_dtype, - variance_dtype=variance_dtype, - compensation_buffer_dtype=compensation_buffer_dtype - ) - - super().__init__( - params, - defaults - ) - - @torch.no_grad() - def step(self, closure=None): - if closure is not None: - with torch.enable_grad(): - closure() - - for group in self.param_groups: - beta1, beta2 = group["betas"] - lr = group["lr"] - weight_decay = group["weight_decay"] - eps = group["eps"] - use_kahan_summation = group["use_kahan_summation"] - momentum_dtype = group["momentum_dtype"] - variance_dtype = group["variance_dtype"] - compensation_buffer_dtype = group["compensation_buffer_dtype"] - - for p in group["params"]: - if p.grad is None: continue - if p.grad.is_sparse: raise RuntimeError - - state = self.state[p] - if len(state) == 0: - state["step"] = torch.tensor(0.0) - state["exp_avg"] = torch.zeros_like(p, dtype=momentum_dtype) - state["exp_avg_sq"] = torch.zeros_like(p, dtype=variance_dtype) - - if use_kahan_summation: - state["compensation"] = torch.zeros_like(p, dtype=compensation_buffer_dtype) - - state["step"] += 1 - step = state["step"] - exp_avg = state["exp_avg"] - exp_avg_sq = state["exp_avg_sq"] - - grad = p.grad - if weight_decay: p.data.mul_(1 - lr * weight_decay) - - exp_avg.mul_(beta1).add_(grad, alpha=1 - beta1) - exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2) - - bias_correction1 = 1 - beta1 ** step - step_size = lr / bias_correction1 - - denom_correction = (1 - beta2**step) ** 0.5 - centered_variance = (exp_avg_sq.sqrt() / denom_correction).add_(eps, alpha=1) - - if use_kahan_summation: - compensation = state["compensation"] - compensation.addcdiv_(exp_avg, centered_variance, value=-step_size) - - temp_buffer = p.detach().clone() - p.data.add_(compensation) - compensation.add_(temp_buffer.sub_(p.data)) - else: p.data.addcdiv_(exp_avg, centered_variance, value=-step_size) \ No newline at end of file diff --git a/main/library/predictors/CREPE/CREPE.py b/main/library/predictors/CREPE/CREPE.py deleted file mode 100644 index ec730c0b332c0c3886fb2f4046cdda6e31edff07..0000000000000000000000000000000000000000 --- a/main/library/predictors/CREPE/CREPE.py +++ /dev/null @@ -1,169 +0,0 @@ -import os -import sys -import torch -import librosa -import scipy.stats - -sys.path.append(os.getcwd()) - -from main.library.algorithm.viterbi import viterbi - -CENTS_PER_BIN, PITCH_BINS, SAMPLE_RATE, WINDOW_SIZE = 20, 360, 16000, 1024 - -class CREPE: - def __init__( - self, - model_path, - model_size="full", - hop_length=512, - batch_size=None, - f0_min=50, - f0_max=1100, - device=None, - sample_rate=16000, - providers=None, - onnx=False, - return_periodicity=False, - compile_model = False, - compile_mode = None - ): - self.device = device - self.hop_length = hop_length - self.batch_size = batch_size - self.sample_rate = sample_rate - self.return_periodicity = return_periodicity - - self.f0_min_bin = (((1200 * torch.tensor(f0_min / 10).log2()) - 1997.3794084376191) / CENTS_PER_BIN).floor().int() - self.f0_max_bin = (((1200 * torch.tensor(f0_max / 10).log2()) - 1997.3794084376191) / CENTS_PER_BIN).ceil().int() - self.eps = torch.tensor(1e-10, device=device) - - if onnx: - import onnxruntime as ort - - sess_options = ort.SessionOptions() - sess_options.log_severity_level = 3 - model = ort.InferenceSession(model_path, sess_options=sess_options, providers=providers) - else: - from main.library.predictors.CREPE.model import CREPEE - - model = CREPEE(model_size) - model.load_state_dict(torch.load(model_path, map_location="cpu", weights_only=True)) - model.to(device).eval() - if compile_model: model = torch.compile(model, mode=compile_mode) - - self.model = model - self.infer = self._infer_onnx if onnx else self._infer_torch - - def bins_to_frequency(self, bins): - if str(bins.device).startswith(("ocl", "privateuseone")): bins = bins.to(torch.float32) - - cents = CENTS_PER_BIN * bins + 1997.3794084376191 - cents = ( - cents + cents.new_tensor( - scipy.stats.triang.rvs( - c=0.5, - loc=-CENTS_PER_BIN, - scale=2 * CENTS_PER_BIN, - size=cents.size() - ) - ) - ) / 1200 - - return 10 * 2 ** cents - - def viterbi(self, logits): - if not hasattr(self, 'transition'): - idx = torch.arange(360, device=logits.device, dtype=logits.dtype) - transition = (12 - (idx[:, None] - idx[None, :]).abs()).clamp(min=0) - self.transition = transition / transition.sum(dim=1, keepdim=True) - - with torch.no_grad(): - probs = torch.nn.functional.softmax(logits, dim=1) - bins = viterbi(probs, self.transition) - - return bins, self.bins_to_frequency(bins) - - def preprocess(self, audio, pad=True): - hop_length = (self.sample_rate // 100) if self.hop_length is None else self.hop_length - - if self.sample_rate != SAMPLE_RATE: - audio = torch.tensor( - librosa.resample( - audio.detach().cpu().numpy().squeeze(0), - orig_sr=self.sample_rate, - target_sr=SAMPLE_RATE, - res_type="soxr_vhq" - ), - device=audio.device - ).unsqueeze(0) - - hop_length = int(hop_length * SAMPLE_RATE / self.sample_rate) - - if pad: - total_frames = 1 + int(audio.size(1) // hop_length) - audio = torch.nn.functional.pad(audio, (WINDOW_SIZE // 2, WINDOW_SIZE // 2)) - else: total_frames = 1 + int((audio.size(1) - WINDOW_SIZE) // hop_length) - - batch_size = total_frames if self.batch_size is None else self.batch_size - - for i in range(0, total_frames, batch_size): - frames = torch.nn.functional.unfold( - audio[:, None, None, max(0, i * hop_length):min(audio.size(1), (i + batch_size - 1) * hop_length + WINDOW_SIZE)], - kernel_size=(1, WINDOW_SIZE), - stride=(1, hop_length) - ).transpose(1, 2) - - if self.device.startswith(("ocl", "privateuseone")): frames = frames.contiguous() - frames = frames.reshape(-1, WINDOW_SIZE).to(self.device) - - frames -= frames.mean(dim=1, keepdim=True) - frames /= self.eps.max(frames.std(dim=1, keepdim=True)) - - yield frames - - def periodicity(self, probabilities, bins): - probs_stacked = probabilities.transpose(1, 2).reshape(-1, PITCH_BINS) - periodicity = probs_stacked.gather(1, bins.reshape(-1, 1).to(torch.int64)) - - return periodicity.reshape(probabilities.size(0), probabilities.size(2)) - - def postprocess(self, probabilities): - probabilities = probabilities.detach() - probabilities[:, :self.f0_min_bin] = -float('inf') - probabilities[:, self.f0_max_bin:] = -float('inf') - - bins, pitch = self.viterbi(probabilities) - - if not self.return_periodicity: return pitch - return pitch, self.periodicity(probabilities, bins) - - def compute_f0(self, audio, pad=True): - results = [] - - with torch.no_grad(): - for frames in self.preprocess(audio, pad): - result = self.postprocess(self.infer(audio, frames)) - results.append((result[0].to(audio.device), result[1].to(audio.device)) if isinstance(result, tuple) else result.to(audio.device)) - - if self.return_periodicity: - pitch, periodicity = zip(*results) - return torch.cat(pitch, 1), torch.cat(periodicity, 1) - - return torch.cat(results, 1) - - def _infer_torch(self, audio, frames): - return self.model( - frames, - embed=False - ).reshape(audio.size(0), -1, PITCH_BINS).transpose(1, 2) - - def _infer_onnx(self, audio, frames): - return torch.tensor( - self.model.run( - [self.model.get_outputs()[0].name], - { - self.model.get_inputs()[0].name: frames.cpu().numpy() - } - )[0].transpose(1, 0)[None], - device=self.device - ) \ No newline at end of file diff --git a/main/library/predictors/CREPE/filter.py b/main/library/predictors/CREPE/filter.py deleted file mode 100644 index 17aff5639a588647bb15c91899d42338425022ad..0000000000000000000000000000000000000000 --- a/main/library/predictors/CREPE/filter.py +++ /dev/null @@ -1,68 +0,0 @@ -import torch - -def mean(signals, win_length=9): - assert signals.dim() == 2 - - signals = signals.unsqueeze(1) - mask = ~torch.isnan(signals) - padding = win_length // 2 - - ones_kernel = torch.ones(signals.size(1), 1, win_length, device=signals.device) - - avg_pooled = torch.nn.functional.conv1d( - torch.where( - mask, - signals, - torch.zeros_like(signals) - ), - ones_kernel, - stride=1, - padding=padding - ) / torch.nn.functional.conv1d( - mask.float(), - ones_kernel, - stride=1, - padding=padding - ).clamp(min=1) - - avg_pooled[avg_pooled == 0] = float("nan") - - return avg_pooled.squeeze(1) - -def median(signals, win_length): - assert signals.dim() == 2 - - signals = signals.unsqueeze(1) - mask = ~torch.isnan(signals) - padding = win_length // 2 - - x = torch.nn.functional.pad( - torch.where( - mask, - signals, - torch.zeros_like(signals) - ), - (padding, padding), - mode="reflect" - ) - - mask = torch.nn.functional.pad( - mask.float(), - (padding, padding), - mode="constant", - value=0 - ) - - x = x.unfold(2, win_length, 1) - mask = mask.unfold(2, win_length, 1) - - x = x.contiguous().view(x.size()[:3] + (-1,)) - mask = mask.contiguous().view(mask.size()[:3] + (-1,)) - - x_sorted, _ = torch.where(mask.bool(), x.float(), float("inf")).to(x).sort(dim=-1) - idx = ((mask.sum(dim=-1) - 1) // 2).clamp(min=0).unsqueeze(-1).long() - - median_pooled = x_sorted.take_along_dim(idx, dim=-1).squeeze(-1) if str(x.device).startswith(("ocl", "privateuseone")) else x_sorted.gather(-1, idx).squeeze(-1) - median_pooled[torch.isinf(median_pooled)] = float("nan") - - return median_pooled.squeeze(1) \ No newline at end of file diff --git a/main/library/predictors/CREPE/model.py b/main/library/predictors/CREPE/model.py deleted file mode 100644 index b88bd598f2db4bc3a782228873d14ae142f63e2c..0000000000000000000000000000000000000000 --- a/main/library/predictors/CREPE/model.py +++ /dev/null @@ -1,47 +0,0 @@ -import torch -import functools - -PITCH_BINS = 360 - -class CREPEE(torch.nn.Module): - def __init__(self, model='full'): - super().__init__() - in_channels = {"full": [1, 1024, 128, 128, 128, 256], "large": [1, 768, 96, 96, 96, 192], "medium": [1, 512, 64, 64, 64, 128], "small": [1, 256, 32, 32, 32, 64], "tiny": [1, 128, 16, 16, 16, 32]}[model] - out_channels = {"full": [1024, 128, 128, 128, 256, 512], "large": [768, 96, 96, 96, 192, 384], "medium": [512, 64, 64, 64, 128, 256], "small": [256, 32, 32, 32, 64, 128], "tiny": [128, 16, 16, 16, 32, 64]}[model] - self.in_features = {"full": 2048, "large": 1536, "medium": 1024, "small": 512, "tiny": 256}[model] - - kernel_sizes = [(512, 1)] + 5 * [(64, 1)] - strides = [(4, 1)] + 5 * [(1, 1)] - batch_norm_fn = functools.partial(torch.nn.BatchNorm2d, eps=0.0010000000474974513, momentum=0.0) - - self.conv1 = torch.nn.Conv2d(in_channels=in_channels[0], out_channels=out_channels[0], kernel_size=kernel_sizes[0], stride=strides[0]) - self.conv1_BN = batch_norm_fn(num_features=out_channels[0]) - - self.conv2 = torch.nn.Conv2d(in_channels=in_channels[1], out_channels=out_channels[1], kernel_size=kernel_sizes[1], stride=strides[1]) - self.conv2_BN = batch_norm_fn(num_features=out_channels[1]) - - self.conv3 = torch.nn.Conv2d(in_channels=in_channels[2], out_channels=out_channels[2], kernel_size=kernel_sizes[2], stride=strides[2]) - self.conv3_BN = batch_norm_fn(num_features=out_channels[2]) - - self.conv4 = torch.nn.Conv2d(in_channels=in_channels[3], out_channels=out_channels[3], kernel_size=kernel_sizes[3], stride=strides[3]) - self.conv4_BN = batch_norm_fn(num_features=out_channels[3]) - - self.conv5 = torch.nn.Conv2d(in_channels=in_channels[4], out_channels=out_channels[4], kernel_size=kernel_sizes[4], stride=strides[4]) - self.conv5_BN = batch_norm_fn(num_features=out_channels[4]) - - self.conv6 = torch.nn.Conv2d(in_channels=in_channels[5], out_channels=out_channels[5], kernel_size=kernel_sizes[5], stride=strides[5]) - self.conv6_BN = batch_norm_fn(num_features=out_channels[5]) - - self.classifier = torch.nn.Linear(in_features=self.in_features, out_features=PITCH_BINS) - - def forward(self, x, embed=False): - x = self.embed(x) - if embed: return x - return self.classifier(self.layer(x, self.conv6, self.conv6_BN).permute(0, 2, 1, 3).reshape(-1, self.in_features)).sigmoid() - - def embed(self, x): - x = x[:, None, :, None] - return self.layer(self.layer(self.layer(self.layer(self.layer(x, self.conv1, self.conv1_BN, (0, 0, 254, 254)), self.conv2, self.conv2_BN), self.conv3, self.conv3_BN), self.conv4, self.conv4_BN), self.conv5, self.conv5_BN) - - def layer(self, x, conv, batch_norm, padding=(0, 0, 31, 32)): - return torch.nn.functional.max_pool2d(batch_norm(torch.nn.functional.relu(conv(torch.nn.functional.pad(x, padding)))), (2, 1), (2, 1)) \ No newline at end of file diff --git a/main/library/predictors/DJCM/DJCM.py b/main/library/predictors/DJCM/DJCM.py deleted file mode 100644 index ed8d5285d135838b7ea957e224d18ff40b911984..0000000000000000000000000000000000000000 --- a/main/library/predictors/DJCM/DJCM.py +++ /dev/null @@ -1,148 +0,0 @@ -import os -import sys -import torch - -import numpy as np - -sys.path.append(os.getcwd()) - -from main.library.predictors.DJCM.spec import Spectrogram - -SAMPLE_RATE, N_CLASS = 16000, 360 - -class DJCM: - def __init__( - self, - model_path, - device = "cpu", - is_half = False, - onnx = False, - svs = False, - providers = ["CPUExecutionProvider"], - batch_size = 1, - segment_len = 5.12, - compile_model = False, - compile_mode = None, - return_tensor = False, - f0_min=50, - f0_max=1100 - ): - super(DJCM, self).__init__() - window_length = 2048 if svs else 1024 - - if onnx: - import onnxruntime as ort - - sess_options = ort.SessionOptions() - sess_options.log_severity_level = 3 - model = ort.InferenceSession(model_path, sess_options=sess_options, providers=providers) - else: - from main.library.predictors.DJCM.model import DJCMM - - model = DJCMM(1, 1, 1, svs=svs, window_length=window_length, n_class=N_CLASS) - model.load_state_dict(torch.load(model_path, map_location="cpu", weights_only=True)) - model.to(device).eval() - - if is_half: model = model.half() - if compile_model: model = torch.compile(model, mode=compile_mode) - - self.model = model - self.f0_min = f0_min - self.f0_max = f0_max - self.device = device - self.batch_size = batch_size - self.seg_len = int(segment_len * SAMPLE_RATE) - self.dtype = torch.float16 if is_half else torch.float32 - self.seg_frames = int(self.seg_len // int(SAMPLE_RATE // 100)) - - self.cents_mapping = np.pad(20 * np.arange(N_CLASS) + 1997.3794084376191, (4, 4)) - self.spec_extractor = Spectrogram(int(SAMPLE_RATE // 100), window_length).to(device) - if return_tensor: self.cents_mapping = torch.as_tensor(self.cents_mapping, dtype=self.dtype, device=device) - - self.offsets = torch.arange(-4, 5, device=device) if return_tensor else np.arange(-4, 5) - self.infer = self._infer_onnx if onnx else (self._infer_torch_fp16 if is_half else self._infer_torch_fp32) - self.to_local_average_cents = self._to_local_average_cents_tensor if return_tensor else self._to_local_average_cents_array - - def infer_from_audio(self, audio, thred=0.03): - if not torch.is_tensor(audio): audio = torch.from_numpy(audio).to(self.device) - if audio.ndim > 1: audio = audio.squeeze() - - with torch.inference_mode(): - segments = self.pad_audio(audio) - - hidden = torch.cat([ - seg[self.seg_frames // 4: int(self.seg_frames * 0.75)] - for seg in torch.cat([ - self.infer(self.spec_extractor(segments[i:i + self.batch_size].float())) - for i in range(0, len(segments), self.batch_size) - ], dim=0) - ], dim=0)[:(audio.shape[-1] // int(SAMPLE_RATE // 100) + 1)].squeeze(0) - - return self.decode(hidden, thred) - - def infer_from_audio_with_pitch(self, audio, thred=0.03): - f0 = self.infer_from_audio(audio, thred) - f0[(f0 < self.f0_min) | (f0 > self.f0_max)] = 0 - - return f0 - - def decode(self, hidden, thred=0.03): - f0 = 10 * (2 ** (self.to_local_average_cents(hidden, thred=thred) / 1200)) - f0[f0 == 10] = 0 - return f0 - - def pad_audio(self, audio): - seg_len = self.seg_len - hop = seg_len // 2 - - audio_len = audio.shape[-1] - left_pad = seg_len // 4 - - return torch.nn.functional.pad( - audio, - (left_pad, (((audio_len + seg_len - 1) // seg_len + 1) * seg_len + hop) - audio_len - left_pad) - ).unfold(0, seg_len, hop).unsqueeze(1).contiguous() - - def _infer_torch_fp32(self, spec): - return self.model(spec) - - def _infer_torchFfp16(self, spec): - return self.model(spec.half()) - - def _infer_onnx(self, spec): - spec = spec.cpu().numpy().astype(np.float32) - - return torch.as_tensor( - self.model.run( - [self.model.get_outputs()[0].name], - {self.model.get_inputs()[0].name: spec} - )[0], - device=self.device, - dtype=self.dtype - ) - - def _to_local_average_cents_array(self, salience, thred=0.05): - salience = salience.cpu().numpy().astype(np.float32) - - center = np.argmax(salience, axis=1) - salience = np.pad(salience, ((0, 0), (4, 4))) - center += 4 - - idx = center[:, None] + self.offsets[None, :] - local_salience = salience[np.arange(salience.shape[0])[:, None], idx] - devided = np.sum(local_salience * self.cents_mapping[idx], axis=1) / np.sum(local_salience, axis=1) - devided[np.max(salience, axis=1) <= thred] = 0 - - return devided - - def _to_local_average_cents_tensor(self, salience, thred=0.05): - center = torch.argmax(salience, dim=1) - salience = torch.nn.functional.pad(salience, (4, 4)) - center += 4 - - idx = center[:, None] + self.offsets[None, :] - local_salience = salience[torch.arange(salience.shape[0], device=salience.device)[:, None], idx] - devided = (local_salience * self.cents_mapping[idx]).sum(dim=1) / local_salience.sum(dim=1) - devided = torch.where(salience.max(dim=1).values <= thred, torch.zeros_like(devided), devided) - - return devided \ No newline at end of file diff --git a/main/library/predictors/DJCM/decoder.py b/main/library/predictors/DJCM/decoder.py deleted file mode 100644 index 0cb3914126f64a6e9e80923a5ed5e4e51e76f841..0000000000000000000000000000000000000000 --- a/main/library/predictors/DJCM/decoder.py +++ /dev/null @@ -1,91 +0,0 @@ -import os -import sys -import torch - -import torch.nn as nn -import torch.nn.functional as F - -sys.path.append(os.getcwd()) - -from main.library.predictors.DJCM.encoder import ResEncoderBlock -from main.library.predictors.DJCM.utils import ResConvBlock, BiGRU, init_bn, init_layer - -class ResDecoderBlock(nn.Module): - def __init__(self, in_channels, out_channels, n_blocks, stride): - super(ResDecoderBlock, self).__init__() - self.conv1 = nn.ConvTranspose2d(in_channels, out_channels, stride, stride, (0, 0), bias=False) - self.bn1 = nn.BatchNorm2d(in_channels, momentum=0.01) - self.conv = nn.ModuleList([ResConvBlock(out_channels * 2, out_channels)]) - - for _ in range(n_blocks - 1): - self.conv.append(ResConvBlock(out_channels, out_channels)) - - self.init_weights() - - def init_weights(self): - init_bn(self.bn1) - init_layer(self.conv1) - - def forward(self, x, concat): - x = self.conv1(F.relu_(self.bn1(x))) - x = torch.cat((x, concat), dim=1) - - for each_layer in self.conv: - x = each_layer(x) - - return x - -class Decoder(nn.Module): - def __init__(self, n_blocks): - super(Decoder, self).__init__() - self.de_blocks = nn.ModuleList([ - ResDecoderBlock(384, 384, n_blocks, (1, 2)), - ResDecoderBlock(384, 384, n_blocks, (1, 2)), - ResDecoderBlock(384, 256, n_blocks, (1, 2)), - ResDecoderBlock(256, 128, n_blocks, (1, 2)), - ResDecoderBlock(128, 64, n_blocks, (1, 2)), - ResDecoderBlock(64, 32, n_blocks, (1, 2)) - ]) - - def forward(self, x, concat_tensors): - for i, layer in enumerate(self.de_blocks): - x = layer(x, concat_tensors[-1 - i]) - - return x - -class PE_Decoder(nn.Module): - def __init__(self, n_blocks, seq_layers=1, window_length = 1024, n_class = 360): - super(PE_Decoder, self).__init__() - self.de_blocks = Decoder(n_blocks) - self.after_conv1 = ResEncoderBlock(32, 32, n_blocks, None) - self.after_conv2 = nn.Conv2d(32, 1, (1, 1)) - self.fc = nn.Sequential( - BiGRU( - (1, window_length // 2), - 1, - seq_layers - ), - nn.Linear( - window_length // 2, - n_class - ), - nn.Sigmoid() - ) - init_layer(self.after_conv2) - - def forward(self, x, concat_tensors): - return self.fc(self.after_conv2(self.after_conv1(self.de_blocks(x, concat_tensors)))).squeeze(1) - -class SVS_Decoder(nn.Module): - def __init__(self, in_channels, n_blocks): - super(SVS_Decoder, self).__init__() - self.de_blocks = Decoder(n_blocks) - self.after_conv1 = ResEncoderBlock(32, 32, n_blocks, None) - self.after_conv2 = nn.Conv2d(32, in_channels * 4, (1, 1)) - self.init_weights() - - def init_weights(self): - init_layer(self.after_conv2) - - def forward(self, x, concat_tensors): - return self.after_conv2(self.after_conv1(self.de_blocks(x, concat_tensors))) \ No newline at end of file diff --git a/main/library/predictors/DJCM/encoder.py b/main/library/predictors/DJCM/encoder.py deleted file mode 100644 index a983469c95c76c7f20361d41bb336e78d869a948..0000000000000000000000000000000000000000 --- a/main/library/predictors/DJCM/encoder.py +++ /dev/null @@ -1,96 +0,0 @@ -import os -import sys - -import torch.nn as nn - -sys.path.append(os.getcwd()) - -from main.library.predictors.DJCM.utils import ResConvBlock - -class ResEncoderBlock(nn.Module): - def __init__( - self, - in_channels, - out_channels, - n_blocks, - kernel_size - ): - super(ResEncoderBlock, self).__init__() - self.conv = nn.ModuleList([ - ResConvBlock( - in_channels, - out_channels - ) - ]) - - for _ in range(n_blocks - 1): - self.conv.append( - ResConvBlock( - out_channels, - out_channels - ) - ) - - self.pool = nn.MaxPool2d(kernel_size) if kernel_size is not None else None - - def forward(self, x): - for each_layer in self.conv: - x = each_layer(x) - - if self.pool is not None: return x, self.pool(x) - return x - -class Encoder(nn.Module): - def __init__( - self, - in_channels, - n_blocks - ): - super(Encoder, self).__init__() - self.en_blocks = nn.ModuleList([ - ResEncoderBlock( - in_channels, - 32, - n_blocks, - (1, 2) - ), - ResEncoderBlock( - 32, - 64, - n_blocks, - (1, 2) - ), - ResEncoderBlock( - 64, - 128, - n_blocks, - (1, 2) - ), - ResEncoderBlock( - 128, - 256, - n_blocks, - (1, 2) - ), - ResEncoderBlock( - 256, - 384, - n_blocks, - (1, 2) - ), - ResEncoderBlock( - 384, - 384, - n_blocks, - (1, 2) - ) - ]) - - def forward(self, x): - concat_tensors = [] - - for layer in self.en_blocks: - _, x = layer(x) - concat_tensors.append(_) - - return x, concat_tensors \ No newline at end of file diff --git a/main/library/predictors/DJCM/model.py b/main/library/predictors/DJCM/model.py deleted file mode 100644 index 91fd225509c8a9849c65a875c221347b38e7ca97..0000000000000000000000000000000000000000 --- a/main/library/predictors/DJCM/model.py +++ /dev/null @@ -1,120 +0,0 @@ -import os -import sys - -import torch.nn as nn - -sys.path.append(os.getcwd()) - -from main.library.predictors.DJCM.utils import init_bn -from main.library.predictors.DJCM.decoder import PE_Decoder, SVS_Decoder -from main.library.predictors.DJCM.encoder import ResEncoderBlock, Encoder - -class LatentBlocks(nn.Module): - def __init__( - self, - n_blocks, - latent_layers - ): - super(LatentBlocks, self).__init__() - self.latent_blocks = nn.ModuleList([ - ResEncoderBlock( - 384, - 384, - n_blocks, - None - ) - for _ in range(latent_layers) - ]) - - def forward(self, x): - for layer in self.latent_blocks: - x = layer(x) - - return x - -class DJCMM(nn.Module): - def __init__( - self, - in_channels, - n_blocks, - latent_layers, - svs=False, - window_length=1024, - n_class=360 - ): - super(DJCMM, self).__init__() - self.bn = nn.BatchNorm2d( - window_length // 2 + 1, - momentum=0.01 - ) - self.pe_encoder = Encoder( - in_channels, - n_blocks - ) - self.pe_latent = LatentBlocks( - n_blocks, - latent_layers - ) - self.pe_decoder = PE_Decoder( - n_blocks, - window_length=window_length, - n_class=n_class - ) - - self.svs = svs - - if svs: - self.svs_encoder = Encoder( - in_channels, - n_blocks - ) - self.svs_latent = LatentBlocks( - n_blocks, - latent_layers - ) - self.svs_decoder = SVS_Decoder( - in_channels, - n_blocks - ) - - init_bn(self.bn) - - def spec(self, x, spec_m): - bs, c, time_steps, freqs_steps = x.shape - x = x.reshape(bs, c // 4, 4, time_steps, freqs_steps) - - mask_spec = x[:, :, 0, :, :].sigmoid() - linear_spec = x[:, :, 3, :, :] - - out_spec = ( - spec_m.detach() * mask_spec + linear_spec - ).relu() - - return out_spec - - def forward(self, spec): - x = self.bn( - spec.transpose(1, 3) - ).transpose(1, 3)[..., :-1] - - if self.svs: - x, concat_tensors = self.svs_encoder(x) - - x = self.svs_decoder( - self.svs_latent(x), - concat_tensors - ) - - x = self.spec( - nn.functional.pad(x, pad=(0, 1)), - spec - )[..., :-1] - - x, concat_tensors = self.pe_encoder(x) - - pe_out = self.pe_decoder( - self.pe_latent(x), - concat_tensors - ) - - return pe_out \ No newline at end of file diff --git a/main/library/predictors/DJCM/spec.py b/main/library/predictors/DJCM/spec.py deleted file mode 100644 index 2d5b6b17ae99784bb13cb4b9e197cf1b3bfed56f..0000000000000000000000000000000000000000 --- a/main/library/predictors/DJCM/spec.py +++ /dev/null @@ -1,60 +0,0 @@ -import os -import sys -import torch - -import numpy as np -import torch.nn as nn - -sys.path.append(os.getcwd()) - -from main.app.variables import config - -class Spectrogram(nn.Module): - def __init__( - self, - hop_length, - win_length, - n_fft=None, - clamp=1e-10 - ): - super(Spectrogram, self).__init__() - self.n_fft = win_length if n_fft is None else n_fft - self.hop_length = hop_length - self.win_length = win_length - self.clamp = clamp - self.register_buffer("window", torch.hann_window(win_length), persistent=False) - self.stftt = self._stft_other_backends if config.device.startswith(("ocl", "privateuseone")) else self._stft_torch - - def forward(self, audio, center=True): - bs, c, segment_samples = audio.shape - - audio = audio.reshape(bs * c, segment_samples) - mag = self.stftt(audio, center).transpose(1, 2).clamp(self.clamp, np.inf) - - return mag.reshape(bs, c, mag.shape[1], mag.shape[2]) - - def _stft_other_backends(self, audio, center=True): - if not hasattr(self, "stft"): - from main.library.backends.utils import STFT - - self.stft = STFT( - filter_length=self.n_fft, - hop_length=self.hop_length, - win_length=self.win_length, - center=center - ).to(audio.device) - - return self.stft.transform(audio, 1e-9) - - def _stft_torch(self, audio, center=True): - fft = torch.stft( - audio, - n_fft=self.n_fft, - hop_length=self.hop_length, - win_length=self.win_length, - window=self.window, - center=center, - pad_mode="reflect", - return_complex=True - ) - return fft.abs() \ No newline at end of file diff --git a/main/library/predictors/DJCM/utils.py b/main/library/predictors/DJCM/utils.py deleted file mode 100644 index 1e803f4127ca9f17dd6ebb7c1fcca115fc2dc39b..0000000000000000000000000000000000000000 --- a/main/library/predictors/DJCM/utils.py +++ /dev/null @@ -1,121 +0,0 @@ -import torch - -from torch import nn -from einops.layers.torch import Rearrange - -def init_layer(layer): - nn.init.xavier_uniform_(layer.weight) - - if hasattr(layer, "bias") and layer.bias is not None: - layer.bias.data.fill_(0.0) - -def init_bn(bn): - bn.bias.data.fill_(0.0) - bn.weight.data.fill_(1.0) - bn.running_mean.data.fill_(0.0) - bn.running_var.data.fill_(1.0) - -class BiGRU(nn.Module): - def __init__( - self, - patch_size, - channels, - depth - ): - super(BiGRU, self).__init__() - patch_width, patch_height = patch_size - patch_dim = channels * patch_height * patch_width - - self.to_patch_embedding = nn.Sequential( - Rearrange( - 'b c (w p1) (h p2) -> b (w h) (p1 p2 c)', - p1=patch_width, - p2=patch_height - ) - ) - - self.gru = nn.GRU( - patch_dim, - patch_dim // 2, - num_layers=depth, - batch_first=True, - bidirectional=True - ) - - def forward(self, x): - x = self.to_patch_embedding(x) - - try: - return self.gru(x)[0] - except: - torch.backends.cudnn.enabled = False - return self.gru(x)[0] - -class ResConvBlock(nn.Module): - def __init__( - self, - in_planes, - out_planes - ): - super(ResConvBlock, self).__init__() - self.bn1 = nn.BatchNorm2d( - in_planes, - momentum=0.01 - ) - self.bn2 = nn.BatchNorm2d( - out_planes, - momentum=0.01 - ) - self.act1 = nn.PReLU() - self.act2 = nn.PReLU() - self.conv1 = nn.Conv2d( - in_planes, - out_planes, - (3, 3), - padding=(1, 1), - bias=False - ) - self.conv2 = nn.Conv2d( - out_planes, - out_planes, - (3, 3), - padding=(1, 1), - bias=False - ) - - self.is_shortcut = False - if in_planes != out_planes: - self.shortcut = nn.Conv2d( - in_planes, - out_planes, - (1, 1) - ) - self.is_shortcut = True - - self.init_weights() - self._return = self.return_shortcut if self.is_shortcut else self.return_non_shortcut - - def init_weights(self): - init_bn(self.bn1) - init_bn(self.bn2) - - init_layer(self.conv1) - init_layer(self.conv2) - - if self.is_shortcut: init_layer(self.shortcut) - - def return_shortcut(self, x, out): - return self.shortcut(x) + out - - def return_non_shortcut(self, x, out): - return out + x - - def forward(self, x): - out = self.conv1( - self.act1(self.bn1(x)) - ) - out = self.conv2( - self.act2(self.bn2(out)) - ) - - return self._return(x, out) \ No newline at end of file diff --git a/main/library/predictors/FCPE/FCPE.py b/main/library/predictors/FCPE/FCPE.py deleted file mode 100644 index 9765e7d109cac5fc3a61236bea369f9217722360..0000000000000000000000000000000000000000 --- a/main/library/predictors/FCPE/FCPE.py +++ /dev/null @@ -1,378 +0,0 @@ -import os -import sys -import torch - -import numpy as np -import torch.nn as nn -import onnxruntime as ort - -from einops import rearrange -from torch.nn.utils.parametrizations import weight_norm - -sys.path.append(os.getcwd()) -os.environ["LRU_CACHE_CAPACITY"] = "3" - -from main.app.variables import config, configs -from main.library.predictors.FCPE.wav2mel import Wav2Mel -from main.library.predictors.FCPE.utils import decrypt_model, DotDict -from main.library.predictors.FCPE.encoder import EncoderLayer, ConformerNaiveEncoder - -@torch.no_grad() -def cent_to_f0(cent): - return 10 * 2 ** (cent / 1200) - -@torch.no_grad() -def f0_to_cent(f0): - return 1200 * (f0 / 10).log2() - -@torch.no_grad() -def latent2cents_local_decoder(cent_table, out_dims, y, threshold = 0.05, mask = True): - B, N, _ = y.size() - ci = cent_table[None, None, :].expand(B, N, -1) - confident, max_index = y.max(dim=-1, keepdim=True) - - local_argmax_index = torch.arange(0, 9).to(max_index.device) + (max_index - 4) - local_argmax_index[local_argmax_index < 0] = 0 - local_argmax_index[local_argmax_index >= out_dims] = out_dims - 1 - - y_l = y.gather(-1, local_argmax_index) - rtn = (ci.gather(-1, local_argmax_index) * y_l).sum(dim=-1, keepdim=True) / y_l.sum(dim=-1, keepdim=True) - - if mask: - confident_mask = torch.ones_like(confident) - confident_mask[confident <= threshold] = float("-INF") - rtn = rtn * confident_mask - - return rtn - -@torch.no_grad() -def latent2cents_local_decoder_cpu(cent_table, out_dims, y, threshold = 0.05, mask = True): - cent_table, y = cent_table.cpu(), y.cpu() - - B, N, _ = y.size() - ci = cent_table[None, None, :].expand(B, N, -1) - confident, max_index = y.max(dim=-1, keepdim=True) - - local_argmax_index = torch.arange(0, 9).to(max_index.device) + (max_index - 4) - local_argmax_index[local_argmax_index < 0] = 0 - local_argmax_index[local_argmax_index >= out_dims] = out_dims - 1 - - y_l = y.gather(-1, local_argmax_index) - rtn = (ci.gather(-1, local_argmax_index) * y_l).sum(dim=-1, keepdim=True) / y_l.sum(dim=-1, keepdim=True) - - if mask: - confident_mask = torch.ones_like(confident) - confident_mask[confident <= threshold] = float("-INF") - rtn = rtn * confident_mask - - return rtn - -def cents_local_decoder(cent_table, y, n_out, confidence, threshold = 0.05, mask=True): - B, N, _ = y.size() - confident, max_index = y.max(dim=-1, keepdim=True) - local_argmax_index = (torch.arange(0, 9).to(max_index.device) + (max_index - 4)).clamp(0, n_out - 1) - y_l = y.gather(-1, local_argmax_index) - rtn = (cent_table[None, None, :].expand(B, N, -1).gather(-1, local_argmax_index) * y_l).sum(dim=-1, keepdim=True) / y_l.sum(dim=-1, keepdim=True) - - if mask: - confident_mask = torch.ones_like(confident) - confident_mask[confident <= threshold] = float("-INF") - rtn = rtn * confident_mask - - return (rtn, confident) if confidence else rtn - -def cents_local_decoder_cpu(cent_table, y, n_out, confidence, threshold = 0.05, mask=True): - cent_table, y = cent_table.cpu(), y.cpu() - - B, N, _ = y.size() - confident, max_index = y.max(dim=-1, keepdim=True) - local_argmax_index = (torch.arange(0, 9).to(max_index.device) + (max_index - 4)).clamp(0, n_out - 1) - y_l = y.gather(-1, local_argmax_index) - rtn = (cent_table[None, None, :].expand(B, N, -1).gather(-1, local_argmax_index) * y_l).sum(dim=-1, keepdim=True) / y_l.sum(dim=-1, keepdim=True) - - if mask: - confident_mask = torch.ones_like(confident) - confident_mask[confident <= threshold] = float("-INF") - rtn = rtn * confident_mask - - return (rtn, confident) if confidence else rtn - -class PCmer(nn.Module): - def __init__( - self, - num_layers, - num_heads, - dim_model, - dim_keys, - dim_values, - residual_dropout, - attention_dropout - ): - super().__init__() - self.num_layers = num_layers - self.num_heads = num_heads - self.dim_model = dim_model - self.dim_values = dim_values - self.dim_keys = dim_keys - self.residual_dropout = residual_dropout - self.attention_dropout = attention_dropout - self._layers = nn.ModuleList([EncoderLayer(self) for _ in range(num_layers)]) - - def forward(self, phone, mask=None): - for layer in self._layers: - phone = layer(phone, mask) - - return phone - -class CFNaiveMelPE(nn.Module): - def __init__( - self, - input_channels, - out_dims, - hidden_dims = 512, - n_layers = 6, - n_heads = 8, - use_fa_norm = False, - conv_only = False, - conv_dropout = 0, - atten_dropout = 0, - ): - super().__init__() - self.input_channels = input_channels - self.out_dims = out_dims - self.hidden_dims = hidden_dims - self.n_layers = n_layers - self.n_heads = n_heads - self.use_fa_norm = use_fa_norm - self.input_stack = nn.Sequential( - nn.Conv1d( - input_channels, - hidden_dims, - 3, - 1, - 1 - ), - nn.GroupNorm( - 4, - hidden_dims - ), - nn.LeakyReLU(), - nn.Conv1d( - hidden_dims, - hidden_dims, - 3, - 1, - 1 - ) - ) - self.net = ConformerNaiveEncoder( - num_layers=n_layers, - num_heads=n_heads, - dim_model=hidden_dims, - use_norm=use_fa_norm, - conv_only=conv_only, - conv_dropout=conv_dropout, - atten_dropout=atten_dropout - ) - self.norm = nn.LayerNorm(hidden_dims) - self.output_proj = weight_norm( - nn.Linear( - hidden_dims, - out_dims - ) - ) - - self.cent_table_b = torch.linspace( - f0_to_cent(torch.Tensor([32.70]))[0], - f0_to_cent(torch.Tensor([1975.5]))[0], - out_dims - ).detach() - self.gaussian_blurred_cent_mask_b = ( - 1200 * torch.Tensor([197.55]).log2() - )[0].detach() - - self.register_buffer("cent_table", self.cent_table_b) - self.register_buffer("gaussian_blurred_cent_mask", self.gaussian_blurred_cent_mask_b) - self.latent2cents_local_decoder = latent2cents_local_decoder_cpu if config.device.startswith("privateuseone") else latent2cents_local_decoder - - def forward(self, mel, threshold = 0.006): - with torch.no_grad(): - mels = rearrange(torch.stack([mel], -1), "B T C K -> (B K) T C") - - x = self.input_stack(mels.transpose(-1, -2)).transpose(-1, -2) - latent = self.output_proj(self.norm(self.net(x))).sigmoid() - - x = cent_to_f0( - self.latent2cents_local_decoder( - self.cent_table, - self.out_dims, - latent, - threshold=threshold - ) - ) - - f0 = rearrange(x, "(B K) T 1 -> B T (K 1)", K=1) - f0 = f0 * (1 - (f0 < 32.70).type(f0.dtype)) - f0[f0 > 1975.5] = 1975.5 - - return f0 - -class CFNaiveMelPE_LEGACY(nn.Module): - def __init__( - self, - input_channel=128, - out_dims=360, - n_layers=12, - n_chans=512, - confidence=False - ): - super().__init__() - self.n_out = out_dims - self.confidence = confidence - - self.cent_table_b = torch.Tensor( - np.linspace( - f0_to_cent(torch.Tensor([32.70]))[0], - f0_to_cent(torch.Tensor([1975.5]))[0], - out_dims - ) - ) - self.register_buffer("cent_table", self.cent_table_b) - - self.stack = nn.Sequential( - nn.Conv1d( - input_channel, - n_chans, - 3, - 1, - 1 - ), - nn.GroupNorm( - 4, - n_chans - ), - nn.LeakyReLU(), - nn.Conv1d( - n_chans, - n_chans, - 3, - 1, - 1 - ) - ) - self.decoder = PCmer( - num_layers=n_layers, - num_heads=8, - dim_model=n_chans, - dim_keys=n_chans, - dim_values=n_chans, - residual_dropout=0.1, - attention_dropout=0.1 - ) - self.norm = nn.LayerNorm(n_chans) - self.dense_out = weight_norm( - nn.Linear( - n_chans, - self.n_out - ) - ) - - self.cents_local_decoder = cents_local_decoder_cpu if config.device.startswith("privateuseone") else cents_local_decoder - - def forward(self, mel, threshold=0.05): - x = self.decoder(self.stack(mel.transpose(1, 2)).transpose(1, 2)) - x = self.dense_out(self.norm(x)).sigmoid() - - x = cent_to_f0( - self.cents_local_decoder( - self.cent_table, - x, - self.n_out, - self.confidence, - threshold=threshold, - mask=True - ) - ) - - return x - -class FCPE: - def __init__( - self, - model_path, - device=None, - threshold=0.05, - providers=None, - onnx=False, - legacy=False, - compile_model=False, - compile_mode=None - ): - self.device = device - self.threshold = threshold - self.wav2mel = Wav2Mel(device=self.device, dtype=torch.float32) - - if onnx: - sess_options = ort.SessionOptions() - sess_options.log_severity_level = 3 - model = ort.InferenceSession(decrypt_model(configs, model_path), sess_options=sess_options, providers=providers) - elif legacy: - ckpt = torch.load(model_path, map_location="cpu", weights_only=True) - self.args = DotDict(ckpt["config"]) - - model = CFNaiveMelPE_LEGACY( - input_channel=self.args.model.input_channel, - out_dims=self.args.model.out_dims, - n_layers=self.args.model.n_layers, - n_chans=self.args.model.n_chans, - confidence=self.args.model.confidence - ) - - model.load_state_dict(ckpt["model"]) - if compile_model: model.decoder, model.stack = torch.compile(model.decoder, mode=compile_mode), torch.compile(model.stack, mode=compile_mode) - model = model.to(self.device).eval() - else: - ckpt = torch.load(model_path, map_location="cpu", weights_only=True) - self.args = DotDict(ckpt["config_dict"]) - - model = CFNaiveMelPE( - input_channels=self.args.mel.num_mels, - out_dims=self.args.model.out_dims, - hidden_dims=self.args.model.hidden_dims, - n_layers=self.args.model.n_layers, - n_heads=self.args.model.n_heads, - use_fa_norm=self.args.model.use_fa_norm, - conv_only=self.args.model.conv_only - ) - - model.load_state_dict(ckpt["model"]) - if compile_model: model.net, model.input_stack = torch.compile(model.net, mode=compile_mode), torch.compile(model.input_stack, mode=compile_mode) - model = model.to(self.device).eval() - - self.model = model - self.infer = self._infer_onnx if onnx else self._infer_torch - - def compute_f0(self, wav): - if not torch.is_tensor(wav): wav = torch.from_numpy(wav).float().to(self.device) - - with torch.no_grad(): - f0 = self.infer(self.wav2mel(audio=wav[None, :]), self.threshold) - f0 = f0[:] if f0.dim() == 1 else f0[0, :, 0] - - return f0 - - def _infer_onnx(self, mel, threshold): - return torch.as_tensor( - self.model.run( - [self.model.get_outputs()[0].name], - { - self.model.get_inputs()[0].name: mel.detach().cpu().numpy(), - self.model.get_inputs()[1].name: np.array(threshold, dtype=np.float32) - } - )[0], - dtype=torch.float32, - device=self.device - ) - - def _infer_torch(self, mel, threshold): - return self.model(mel, threshold=threshold) \ No newline at end of file diff --git a/main/library/predictors/FCPE/attentions.py b/main/library/predictors/FCPE/attentions.py deleted file mode 100644 index 94f4eee38e26a5a857749f5224e6bc99393590ee..0000000000000000000000000000000000000000 --- a/main/library/predictors/FCPE/attentions.py +++ /dev/null @@ -1,448 +0,0 @@ -import math -import torch - -import torch.nn.functional as F - -from torch import nn, einsum -from functools import partial -from einops import rearrange, repeat, pack, unpack - -def exists(val): - return val is not None - -def default(value, d): - return value if exists(value) else d - -def empty(tensor): - return tensor.numel() == 0 - -def pad_to_multiple(tensor, multiple, dim=-1, value=0): - seqlen = tensor.shape[dim] - m = seqlen / multiple - - if m.is_integer(): return False, tensor - return True, F.pad(tensor, (*((0,) * (-1 - dim) * 2), 0, (math.ceil(m) * multiple - seqlen)), value = value) - -def look_around(x, backward = 1, forward = 0, pad_value = -1, dim = 2): - t = x.shape[1] - dims = (len(x.shape) - dim) * (0, 0) - padded_x = F.pad(x, (*dims, backward, forward), value = pad_value) - - return torch.cat([padded_x[:, ind:(ind + t), ...] for ind in range(forward + backward + 1)], dim = dim) - -def rotate_half(x): - x1, x2 = rearrange(x, 'b ... (r d) -> b ... r d', r = 2).unbind(dim = -2) - return torch.cat((-x2, x1), dim = -1) - -def apply_rotary_pos_emb(q, k, freqs, scale = 1): - q_len = q.shape[-2] - q_freqs = freqs[..., -q_len:, :] - - inv_scale = scale ** -1 - if scale.ndim == 2: scale = scale[-q_len:, :] - - q = (q * q_freqs.cos() * scale) + (rotate_half(q) * q_freqs.sin() * scale) - k = (k * freqs.cos() * inv_scale) + (rotate_half(k) * freqs.sin() * inv_scale) - - return q, k - -def orthogonal_matrix_chunk(cols, qr_uniform_q=False, device=None): - unstructured_block = torch.randn((cols, cols), device=device) - - q, r = torch.linalg.qr(unstructured_block.cpu(), mode="reduced") - q, r = map(lambda t: t.to(device), (q, r)) - - if qr_uniform_q: - d = r.diag(0) - q *= d.sign() - - return q.t() - -def gaussian_orthogonal_random_matrix(nb_rows, nb_columns, scaling=0, qr_uniform_q=False, device=None): - nb_full_blocks = int(nb_rows / nb_columns) - block_list = [] - - for _ in range(nb_full_blocks): - block_list.append(orthogonal_matrix_chunk(nb_columns, qr_uniform_q=qr_uniform_q, device=device)) - - remaining_rows = nb_rows - nb_full_blocks * nb_columns - - if remaining_rows > 0: - block_list.append( - orthogonal_matrix_chunk( - nb_columns, - qr_uniform_q=qr_uniform_q, - device=device - )[:remaining_rows] - ) - if scaling == 0: - multiplier = torch.randn( - (nb_rows, nb_columns), - device=device - ).norm(dim=1) - elif scaling == 1: - multiplier = math.sqrt( - (float(nb_columns)) - ) * torch.ones( - (nb_rows,), - device=device - ) - else: raise ValueError(f"{scaling} != 0, 1") - - return multiplier.diag() @ torch.cat(block_list) - -def linear_attention(q, k, v): - return einsum( - "...ed,...nd->...ne", - k, - q - ) if v is None else einsum( - "...de,...nd,...n->...ne", - einsum( - "...nd,...ne->...de", - k, - v - ), - q, - 1.0 / (einsum( - "...nd,...d->...n", - q, - k.sum(dim=-2).type_as(q) - ) + 1e-8) - ) - -def softmax_kernel(data, *, projection_matrix, is_query, normalize_data=True, eps=1e-4, device=None): - b, h, *_ = data.shape - - data_normalizer = (data.shape[-1] ** -0.25) if normalize_data else 1.0 - ratio = projection_matrix.shape[0] ** -0.5 - - data_dash = torch.einsum("...id,...jd->...ij", (data_normalizer * data), repeat(projection_matrix, "j d -> b h j d", b=b, h=h).type_as(data)) - diag_data = (((data**2).sum(dim=-1) / 2.0) * (data_normalizer**2)).unsqueeze(dim=-1) - - return (ratio * ((data_dash - diag_data - data_dash.max(dim=-1, keepdim=True).values).exp() + eps) if is_query else ratio * ((data_dash - diag_data + eps).exp())).type_as(data) - -class SinusoidalEmbeddings(nn.Module): - def __init__( - self, - dim, - scale_base = None, - use_xpos = False, - theta = 10000 - ): - super().__init__() - inv_freq = 1. / (theta ** (torch.arange(0, dim, 2).float() / dim)) - self.register_buffer('inv_freq', inv_freq) - self.use_xpos = use_xpos - self.scale_base = scale_base - assert not (use_xpos and not exists(scale_base)) - scale = (torch.arange(0, dim, 2) + 0.4 * dim) / (1.4 * dim) - self.register_buffer('scale', scale, persistent = False) - - def forward(self, x): - seq_len, device = x.shape[-2], x.device - t = torch.arange(seq_len, device = x.device).type_as(self.inv_freq) - - freqs = torch.einsum('i , j -> i j', t, self.inv_freq) - freqs = torch.cat((freqs, freqs), dim = -1) - - if not self.use_xpos: return freqs, torch.ones(1, device = device) - - power = (t - (seq_len // 2)) / self.scale_base - scale = self.scale ** rearrange(power, 'n -> n 1') - - return freqs, torch.cat((scale, scale), dim = -1) - -class LocalAttention(nn.Module): - def __init__( - self, - window_size, - causal = False, - look_backward = 1, - look_forward = None, - dropout = 0., - shared_qk = False, - rel_pos_emb_config = None, - dim = None, - autopad = False, - exact_windowsize = False, - scale = None, - use_rotary_pos_emb = True, - use_xpos = False, - xpos_scale_base = None - ): - super().__init__() - look_forward = default(look_forward, 0 if causal else 1) - assert not (causal and look_forward > 0) - self.scale = scale - self.window_size = window_size - self.autopad = autopad - self.exact_windowsize = exact_windowsize - self.causal = causal - self.look_backward = look_backward - self.look_forward = look_forward - self.dropout = nn.Dropout(dropout) - self.shared_qk = shared_qk - self.rel_pos = None - self.use_xpos = use_xpos - if use_rotary_pos_emb and (exists(rel_pos_emb_config) or exists(dim)): - if exists(rel_pos_emb_config): dim = rel_pos_emb_config[0] - self.rel_pos = SinusoidalEmbeddings(dim, use_xpos = use_xpos, scale_base = default(xpos_scale_base, window_size // 2)) - - def forward(self, q, k, v, mask = None, input_mask = None, attn_bias = None, window_size = None): - mask = default(mask, input_mask) - assert not (exists(window_size) and not self.use_xpos) - - ( - _, - autopad, - pad_value, - window_size, - causal, - look_backward, - look_forward, - shared_qk - ) = ( - q.shape, - self.autopad, - -1, - default( - window_size, - self.window_size - ), - self.causal, - self.look_backward, - self.look_forward, - self.shared_qk - ) - - (q, packed_shape), (k, _), (v, _) = map(lambda t: pack([t], '* n d'), (q, k, v)) - - if autopad: - orig_seq_len = q.shape[1] - (_, q), (_, k), (_, v) = map(lambda t: pad_to_multiple(t, self.window_size, dim = -2), (q, k, v)) - - b, n, dim_head, device, dtype = *q.shape, q.device, q.dtype - scale = default(self.scale, dim_head ** -0.5) - - assert (n % window_size) == 0 - windows = n // window_size - - if shared_qk: k = F.normalize(k, dim = -1).type(k.dtype) - - seq = torch.arange(n, device = device) - b_t = rearrange(seq, '(w n) -> 1 w n', w = windows, n = window_size) - bq, bk, bv = map(lambda t: rearrange(t, 'b (w n) d -> b w n d', w = windows), (q, k, v)) - - bq = bq * scale - look_around_kwargs = dict(backward = look_backward, forward = look_forward, pad_value = pad_value) - - bk = look_around(bk, **look_around_kwargs) - bv = look_around(bv, **look_around_kwargs) - - if exists(self.rel_pos): - pos_emb, xpos_scale = self.rel_pos(bk) - bq, bk = apply_rotary_pos_emb(bq, bk, pos_emb, scale = xpos_scale) - - bq_t = b_t - bq_k = look_around(b_t, **look_around_kwargs) - bq_t = rearrange(bq_t, '... i -> ... i 1') - bq_k = rearrange(bq_k, '... j -> ... 1 j') - - pad_mask = bq_k == pad_value - sim = einsum('b h i e, b h j e -> b h i j', bq, bk) - - if exists(attn_bias): - heads = attn_bias.shape[0] - assert (b % heads) == 0 - - attn_bias = repeat(attn_bias, 'h i j -> (b h) 1 i j', b = b // heads) - sim = sim + attn_bias - - mask_value = -torch.finfo(sim.dtype).max - if shared_qk: - self_mask = bq_t == bq_k - sim = sim.masked_fill(self_mask, -5e4) - del self_mask - - if causal: - causal_mask = bq_t < bq_k - if self.exact_windowsize: causal_mask = causal_mask | (bq_t > (bq_k + (self.window_size * self.look_backward))) - sim = sim.masked_fill(causal_mask, mask_value) - del causal_mask - - sim = sim.masked_fill( - ((bq_k - (self.window_size * self.look_forward)) > bq_t) | (bq_t > (bq_k + (self.window_size * self.look_backward))) | pad_mask, - mask_value - ) if not causal and self.exact_windowsize else sim.masked_fill( - pad_mask, - mask_value - ) - - if exists(mask): - batch = mask.shape[0] - assert (b % batch) == 0 - - h = b // mask.shape[0] - if autopad: _, mask = pad_to_multiple(mask, window_size, dim = -1, value = False) - - mask = repeat( - rearrange( - look_around( - rearrange( - mask, - '... (w n) -> (...) w n', - w = windows, - n = window_size - ), - **{ - **look_around_kwargs, - 'pad_value': False - } - ), - '... j -> ... 1 j' - ), - 'b ... -> (b h) ...', - h = h - ) - - sim = sim.masked_fill(~mask, mask_value) - del mask - - out = rearrange( - einsum( - 'b h i j, b h j e -> b h i e', - self.dropout(sim.softmax(dim = -1)), - bv - ), - 'b w n d -> b (w n) d' - ) - - if autopad: out = out[:, :orig_seq_len, :] - out, *_ = unpack(out, packed_shape, '* n d') - - return out - -class FastAttention(nn.Module): - def __init__( - self, - dim_heads, - nb_features=None, - ortho_scaling=0, - causal=False, - generalized_attention=False, - kernel_fn=nn.ReLU(), - qr_uniform_q=False, - no_projection=False - ): - super().__init__() - nb_features = default(nb_features, int(dim_heads * math.log(dim_heads))) - self.dim_heads = dim_heads - self.nb_features = nb_features - self.ortho_scaling = ortho_scaling - self.create_projection = partial( - gaussian_orthogonal_random_matrix, - nb_rows=self.nb_features, - nb_columns=dim_heads, - scaling=ortho_scaling, - qr_uniform_q=qr_uniform_q - ) - projection_matrix = self.create_projection() - self.register_buffer("projection_matrix", projection_matrix) - self.generalized_attention = generalized_attention - self.kernel_fn = kernel_fn - self.no_projection = no_projection - self.causal = causal - - @torch.no_grad() - def redraw_projection_matrix(self): - projections = self.create_projection() - self.projection_matrix.copy_(projections) - del projections - - def forward(self, q, k, v): - if self.no_projection: q, k = q.softmax(dim=-1), (k.exp() if self.causal else k.softmax(dim=-2)) - else: - create_kernel = partial(softmax_kernel, projection_matrix=self.projection_matrix, device=q.device) - q, k = create_kernel(q, is_query=True), create_kernel(k, is_query=False) - - attn_fn = linear_attention if not self.causal else self.causal_linear_fn - return attn_fn(q, k, None) if v is None else attn_fn(q, k, v) - -class SelfAttention(nn.Module): - def __init__( - self, - dim, - causal=False, - heads=8, - dim_head=64, - local_heads=0, - local_window_size=256, - nb_features=None, - feature_redraw_interval=1000, - generalized_attention=False, - kernel_fn=nn.ReLU(), - qr_uniform_q=False, - dropout=0.0, - no_projection=False - ): - super().__init__() - assert dim % heads == 0 - dim_head = default(dim_head, dim // heads) - inner_dim = dim_head * heads - self.fast_attention = FastAttention( - dim_head, - nb_features, - causal=causal, - generalized_attention=generalized_attention, - kernel_fn=kernel_fn, - qr_uniform_q=qr_uniform_q, - no_projection=no_projection - ) - self.heads = heads - self.global_heads = heads - local_heads - self.local_attn = ( - LocalAttention( - window_size=local_window_size, - causal=causal, - autopad=True, - dropout=dropout, - look_forward=int(not causal), - rel_pos_emb_config=(dim_head, local_heads) - ) if local_heads > 0 else None - ) - self.to_q = nn.Linear(dim, inner_dim) - self.to_k = nn.Linear(dim, inner_dim) - self.to_v = nn.Linear(dim, inner_dim) - self.to_out = nn.Linear(inner_dim, dim) - self.dropout = nn.Dropout(dropout) - - @torch.no_grad() - def redraw_projection_matrix(self): - self.fast_attention.redraw_projection_matrix() - - def forward(self, x, context=None, mask=None, context_mask=None, name=None, inference=False, **kwargs): - _, _, _, h, gh = *x.shape, self.heads, self.global_heads - cross_attend = exists(context) - context = default(context, x) - context_mask = default(context_mask, mask) if not cross_attend else context_mask - - q, k, v = map(lambda t: rearrange(t, "b n (h d) -> b h n d", h=h), (self.to_q(x), self.to_k(context), self.to_v(context))) - (q, lq), (k, lk), (v, lv) = map(lambda t: (t[:, :gh], t[:, gh:]), (q, k, v)) - - attn_outs = [] - - if not empty(q): - if exists(context_mask): v.masked_fill_(~context_mask[:, None, :, None], 0.0) - if cross_attend: pass - else: out = self.fast_attention(q, k, v) - - attn_outs.append(out) - - if not empty(lq): - assert not cross_attend - - out = self.local_attn(lq, lk, lv, input_mask=mask) - attn_outs.append(out) - - return self.dropout(self.to_out(rearrange(torch.cat(attn_outs, dim=1), "b h n d -> b n (h d)"))) \ No newline at end of file diff --git a/main/library/predictors/FCPE/encoder.py b/main/library/predictors/FCPE/encoder.py deleted file mode 100644 index 1a6f97a6293bf29ac2f5f3d1a6e062fa3644d6cf..0000000000000000000000000000000000000000 --- a/main/library/predictors/FCPE/encoder.py +++ /dev/null @@ -1,183 +0,0 @@ -import os -import sys - -import torch.nn as nn -import torch.nn.functional as F - -sys.path.append(os.getcwd()) - -from main.library.predictors.FCPE.attentions import SelfAttention -from main.library.predictors.FCPE.utils import calc_same_padding, Transpose, GLU, Swish - -class ConformerConvModule_LEGACY(nn.Module): - def __init__( - self, - dim, - causal=False, - expansion_factor=2, - kernel_size=31, - dropout=0.0 - ): - super().__init__() - inner_dim = dim * expansion_factor - self.net = nn.Sequential( - nn.LayerNorm(dim), - Transpose((1, 2)), - nn.Conv1d(dim, inner_dim * 2, 1), - GLU(dim=1), - DepthWiseConv1d_LEGACY( - inner_dim, - inner_dim, - kernel_size=kernel_size, - padding=( - calc_same_padding(kernel_size) if not causal else (kernel_size - 1, 0) - ) - ), - Swish(), - nn.Conv1d(inner_dim, dim, 1), - Transpose((1, 2)), - nn.Dropout(dropout) - ) - - def forward(self, x): - return self.net(x) - -class ConformerConvModule(nn.Module): - def __init__( - self, - dim, - expansion_factor=2, - kernel_size=31, - dropout=0 - ): - super().__init__() - inner_dim = dim * expansion_factor - self.net = nn.Sequential( - nn.LayerNorm(dim), - Transpose((1, 2)), - nn.Conv1d(dim, inner_dim * 2, 1), - nn.GLU(dim=1), - DepthWiseConv1d( - inner_dim, - inner_dim, - kernel_size=kernel_size, - padding=calc_same_padding(kernel_size)[0], - groups=inner_dim - ), - nn.SiLU(), - nn.Conv1d(inner_dim, dim, 1), - Transpose((1, 2)), - nn.Dropout(dropout) - ) - - def forward(self, x): - return self.net(x) - -class DepthWiseConv1d_LEGACY(nn.Module): - def __init__( - self, - chan_in, - chan_out, - kernel_size, - padding - ): - super().__init__() - self.padding = padding - self.conv = nn.Conv1d(chan_in, chan_out, kernel_size, groups=chan_in) - - def forward(self, x): - return self.conv(F.pad(x, self.padding)) - -class DepthWiseConv1d(nn.Module): - def __init__( - self, - chan_in, - chan_out, - kernel_size, - padding, - groups - ): - super().__init__() - self.conv = nn.Conv1d(chan_in, chan_out, kernel_size=kernel_size, padding=padding, groups=groups) - - def forward(self, x): - return self.conv(x) - -class EncoderLayer(nn.Module): - def __init__( - self, - parent - ): - super().__init__() - self.conformer = ConformerConvModule_LEGACY(parent.dim_model) - self.norm = nn.LayerNorm(parent.dim_model) - self.dropout = nn.Dropout(parent.residual_dropout) - self.attn = SelfAttention(dim=parent.dim_model, heads=parent.num_heads, causal=False) - - def forward(self, phone, mask=None): - phone = phone + (self.attn(self.norm(phone), mask=mask)) - return phone + (self.conformer(phone)) - -class ConformerNaiveEncoder(nn.Module): - def __init__( - self, - num_layers, - num_heads, - dim_model, - use_norm = False, - conv_only = False, - conv_dropout = 0, - atten_dropout = 0 - ): - super().__init__() - self.num_layers = num_layers - self.num_heads = num_heads - self.dim_model = dim_model - self.use_norm = use_norm - self.residual_dropout = 0.1 - self.attention_dropout = 0.1 - self.encoder_layers = nn.ModuleList([ - CFNEncoderLayer(dim_model, num_heads, use_norm, conv_only, conv_dropout, atten_dropout) - for _ in range(num_layers) - ]) - - def forward(self, x, mask=None): - for layer in self.encoder_layers: - x = layer(x, mask) - - return x - -class CFNEncoderLayer(nn.Module): - def __init__( - self, - dim_model, - num_heads = 8, - use_norm = False, - conv_only = False, - conv_dropout = 0, - atten_dropout = 0 - ): - super().__init__() - self.conformer = ( - nn.Sequential( - ConformerConvModule(dim_model), - nn.Dropout(conv_dropout) - ) - ) if conv_dropout > 0 else ( - ConformerConvModule(dim_model) - ) - - self.norm = nn.LayerNorm(dim_model) - self.dropout = nn.Dropout(0.1) - - self.attn = SelfAttention( - dim=dim_model, - heads=num_heads, - causal=False, - use_norm=use_norm, - dropout=atten_dropout - ) if not conv_only else None - - def forward(self, x, mask=None): - if self.attn is not None: x = x + (self.attn(self.norm(x), mask=mask)) - return x + (self.conformer(x)) \ No newline at end of file diff --git a/main/library/predictors/FCPE/stft.py b/main/library/predictors/FCPE/stft.py deleted file mode 100644 index 2db2fd1bfd274d30479da02667ceeb11542f084f..0000000000000000000000000000000000000000 --- a/main/library/predictors/FCPE/stft.py +++ /dev/null @@ -1,64 +0,0 @@ -import os -import sys -import torch - -import torch.nn.functional as F - -sys.path.append(os.getcwd()) - -from main.app.variables import config -from main.library.audio.features import mel - -class STFT: - def __init__( - self, - sr=22050, - n_mels=80, - n_fft=1024, - win_size=1024, - hop_length=256, - fmin=20, - fmax=11025, - clip_val=1e-5 - ): - self.n_fft = n_fft - self.clip_val = clip_val - self.win_size = win_size - self.hop_length = hop_length - self.hann_window = torch.hann_window(self.win_size).to(config.device) - self.stftt = self._stft_other_backends if config.device.startswith(("ocl", "privateuseone")) else self._stft_torch - self.mel_basis = mel(sr=sr, n_fft=self.n_fft, n_mels=n_mels, fmin=fmin, fmax=fmax, device=config.device) - - def get_mel(self, y, center=False): - pad_left = (self.win_size - self.hop_length) // 2 - pad_right = max((self.win_size - self.hop_length + 1) // 2, self.win_size - y.size(-1) - pad_left) - - return (self.mel_basis @ self.stftt(F.pad(y.unsqueeze(1), (pad_left, pad_right), mode="reflect" if pad_right < y.size(-1) else "constant").squeeze(1), center)).clamp(min=self.clip_val).log() - - def _stft_other_backends(self, pad, center=True): - if not hasattr(self, "stft"): - from main.library.backends.utils import STFT as _STFT - - self.stft = _STFT( - filter_length=self.n_fft, - hop_length=self.hop_length, - win_length=self.win_size, - ).to(pad.device) - - return self.stft.transform(pad, center=center, eps=1e-5) - - def _stft_torch(self, pad, center=True): - spec = torch.stft( - pad, - n_fft=self.n_fft, - hop_length=self.hop_length, - win_length=self.win_size, - window=self.hann_window, - center=center, - pad_mode="reflect", - normalized=False, - onesided=True, - return_complex=True - ) - - return spec.abs().clamp_min_(1e-9) \ No newline at end of file diff --git a/main/library/predictors/FCPE/utils.py b/main/library/predictors/FCPE/utils.py deleted file mode 100644 index 23083e914b000c0d350c7fdfd104ebf9d5d0ead1..0000000000000000000000000000000000000000 --- a/main/library/predictors/FCPE/utils.py +++ /dev/null @@ -1,78 +0,0 @@ -import os -import torch - -from torch import nn -from io import BytesIO -from Crypto.Cipher import AES -from Crypto.Util.Padding import unpad - -def decrypt_model(configs, input_path): - with open(input_path, "rb") as f: - data = f.read() - - with open( - os.path.join(configs["binary_path"], "decrypt.bin"), - "rb" - ) as f: - key = f.read() - - return BytesIO( - unpad( - AES.new( - key, - AES.MODE_CBC, - data[:16] - ).decrypt(data[16:]), - AES.block_size - ) - ).read() - -def calc_same_padding(kernel_size): - pad = kernel_size // 2 - return (pad, pad - (kernel_size + 1) % 2) - -def torch_interp(x, xp, fp): - sort_idx = xp.argsort() - xp = xp[sort_idx] - fp = fp[sort_idx] - - right_idxs = torch.searchsorted(xp, x).clamp(max=len(xp) - 1) - left_idxs = (right_idxs - 1).clamp(min=0) - x_left = xp[left_idxs] - y_left = fp[left_idxs] - - interp_vals = y_left + ((x - x_left) * (fp[right_idxs] - y_left) / (xp[right_idxs] - x_left)) - interp_vals[x < xp[0]] = fp[0] - interp_vals[x > xp[-1]] = fp[-1] - - return interp_vals - -class DotDict(dict): - def __getattr__(*args): - val = dict.get(*args) - return DotDict(val) if type(val) is dict else val - - __setattr__ = dict.__setitem__ - __delattr__ = dict.__delitem__ - -class Swish(nn.Module): - def forward(self, x): - return x * x.sigmoid() - -class Transpose(nn.Module): - def __init__(self, dims): - super().__init__() - assert len(dims) == 2 - self.dims = dims - - def forward(self, x): - return x.transpose(*self.dims) - -class GLU(nn.Module): - def __init__(self, dim): - super().__init__() - self.dim = dim - - def forward(self, x): - out, gate = x.chunk(2, dim=self.dim) - return out * gate.sigmoid() \ No newline at end of file diff --git a/main/library/predictors/FCPE/wav2mel.py b/main/library/predictors/FCPE/wav2mel.py deleted file mode 100644 index 989d2d219fe4145ca2015fe1b309f2bfbf00c906..0000000000000000000000000000000000000000 --- a/main/library/predictors/FCPE/wav2mel.py +++ /dev/null @@ -1,28 +0,0 @@ -import os -import sys -import torch - -sys.path.append(os.getcwd()) - -from main.library.predictors.FCPE.stft import STFT - -class Wav2Mel: - def __init__( - self, - device=None, - dtype=torch.float32 - ): - self.sample_rate = 16000 - self.hop_size = 160 - self.device = device - self.dtype = dtype - self.stft = STFT(16000, 128, 1024, 1024, 160, 0, 8000) - - def __call__(self, audio): - audio = audio.to(self.dtype).to(self.device) - mel = self.stft.get_mel(audio).transpose(1, 2) - - n_frames = int(audio.shape[1] // self.hop_size) + 1 - mel = (torch.cat((mel, mel[:, -1:, :]), 1) if n_frames > int(mel.shape[1]) else mel) - - return mel[:, :n_frames, :] if n_frames < int(mel.shape[1]) else mel \ No newline at end of file diff --git a/main/library/predictors/Generator.py b/main/library/predictors/Generator.py deleted file mode 100644 index dd8ef1eaee96cd9d475e34fa06b40269d85a6bf7..0000000000000000000000000000000000000000 --- a/main/library/predictors/Generator.py +++ /dev/null @@ -1,836 +0,0 @@ -import os -import re -import sys -import math -import torch -import parselmouth - -import numba as nb -import numpy as np - -from scipy.signal import medfilt -from librosa import yin, pyin, piptrack - -sys.path.append(os.getcwd()) - -from main.library.utils import circular_write -from main.library.predictors.CREPE.filter import mean, median -from main.library.predictors.WORLD.SWIPE import swipe, stonemask -from main.app.variables import config, configs, logger, translations - -NOTE = [49.00, 51.91, 55.00, 58.27, 61.74, 65.41, 69.30, 73.42, 77.78, 82.41, 87.31, 92.50, 98.00, 103.83, 110.00, 116.54, 123.47, 130.81, 138.59, 146.83, 155.56, 164.81, 174.61, 185.00, 196.00, 207.65, 220.00, 233.08, 246.94, 261.63, 277.18, 293.66, 311.13, 329.63, 349.23, 369.99, 392.00, 415.30, 440.00, 466.16, 493.88, 523.25, 554.37, 587.33, 622.25, 659.25, 698.46, 739.99, 783.99, 830.61, 880.00, 932.33, 987.77, 1046.50] - -def medfilts(x, kernel_size = 3): - if kernel_size % 2 == 0: raise ValueError - if x.ndim != 1: raise ValueError - - if torch.is_tensor(x): - return torch.nn.functional.pad( - x.unsqueeze(0).unsqueeze(0), - (kernel_size // 2, kernel_size // 2), - mode="constant", - value=0.0 - ).squeeze(0).squeeze(0).unfold(0, kernel_size, 1).median(dim=-1).values - else: - return medfilt(x, kernel_size) - -def autotune_f0(note_dict, f0, f0_autotune_strength): - if torch.is_tensor(f0): - notes = torch.as_tensor(note_dict, dtype=f0.dtype, device=f0.device) - nearest = notes[torch.cdist(f0[:, None], notes[:, None]).argmin(dim=1)] - else: - notes = np.asarray(note_dict, dtype=f0.dtype) - nearest = notes[np.abs(f0[:, None] - notes).argmin(axis=1)] - - return f0 + (nearest - f0) * f0_autotune_strength - - -def extract_median_f0(f0): - f0 = np.where(f0 == 0, np.nan, f0) - - return float( - np.median( - np.interp( - np.arange(len(f0)), - np.where(~np.isnan(f0))[0], - f0[~np.isnan(f0)] - ) - ) - ) - -def proposal_f0_up_key(f0, target_f0 = 155.0, limit = 12): - if torch.is_tensor(f0): f0 = f0.cpu().numpy() - - try: - return max( - -limit, - min( - limit, int(np.round(12 * np.log2(target_f0 / extract_median_f0(f0)))) - ) - ) - except ValueError: - return 0 - -@nb.jit(nopython=True) -def post_process( - f0, - f0_up_key, - manual_x_pad, - f0_mel_min, - f0_mel_max, - manual_f0 = None -): - f0 *= pow(2, f0_up_key / 12) - - if manual_f0 is not None: - start_frame = int(manual_x_pad * 100) - - replace_f0 = np.interp( - np.linspace( - manual_f0[:, 0][0], - manual_f0[:, 0][-1], - len(manual_f0[:, 0]) - ), - manual_f0[:, 0], - manual_f0[:, 1] - ) - - end_frame = start_frame + len(replace_f0) - if end_frame > len(f0): - end_frame = len(f0) - replace_f0 = replace_f0[:end_frame - start_frame] - - f0[start_frame:end_frame] = replace_f0 - - f0_mel = 1127 * np.log(1 + f0 / 700) - f0_mel[f0_mel > 0] = (f0_mel[f0_mel > 0] - f0_mel_min) * 254 / (f0_mel_max - f0_mel_min) + 1 - f0_mel[f0_mel <= 1] = 1 - f0_mel[f0_mel > 255] = 255 - - return np.rint(f0_mel).astype(np.int32), f0 - -def realtime_post_process( - f0, - pitch, - pitchf, - f0_up_key = 0, - f0_min = 50.0, - f0_max = 1100.0 -): - f0 *= 2 ** (f0_up_key / 12) - - f0_mel = 1127.0 * (1.0 + f0 / 700.0).log() - f0_mel = torch.clip((f0_mel - f0_min) * 254 / (f0_max - f0_min) + 1, 1, 255, out=f0_mel) - f0_coarse = torch.round(f0_mel, out=f0_mel).long() - - if pitch is not None and pitchf is not None: - f0_coarse = f0_coarse[3:-1] if f0_coarse.shape[0] > 4 else f0_coarse - f0 = f0[3:-1] if f0.shape[0] > 4 else f0 - - circular_write(f0_coarse, pitch) - circular_write(f0, pitchf) - else: - pitch = f0_coarse - pitchf = f0 - - return pitch.unsqueeze(0), pitchf.unsqueeze(0) - -class Generator: - def __init__( - self, - sample_rate = 16000, - hop_length = 160, - f0_min = 50, - f0_max = 1100, - alpha = 0.5, - is_half = False, - device = "cpu", - predictor_onnx = False, - return_tensor = False - ): - self.alpha = alpha - self.f0_min = f0_min - self.f0_max = f0_max - self.hop_length = hop_length - self.sample_rate = sample_rate - self.f0_mel_min = 1127 * math.log(1 + self.f0_min / 700) - self.f0_mel_max = 1127 * math.log(1 + self.f0_max / 700) - - self.window = 160 - self.batch_size = 512 - self.ref_freqs = NOTE - self.frame_period = self.window / self.sample_rate * 1000 - self.time_step = self.frame_period / 1000 - - self.cache_args = None - self.cache_flags = None - self.cache_f0_method = None - self.providers = config.providers - self.is_half = is_half - self.device = device - - self.compile_model = config.compile_all - self.compile_mode = config.compile_mode - self.predictor_onnx = predictor_onnx - self.return_tensor = return_tensor - - self.resize = configs.get("turn_on_resize_f0_for_all", False) - self._resize_f0 = self._resize_tensor_f0 if self.return_tensor else self._resize_array_f0 - self.resize_dtype = torch.float64 if self.device.startswith(("cpu", "cuda")) else torch.float32 - - def calculator( - self, - x_pad, - f0_method, - x, - f0_up_key = 0, - p_len = None, - filter_radius = 3, - f0_autotune = False, - f0_autotune_strength = 1, - manual_f0 = None, - proposal_pitch = False, - proposal_pitch_threshold = 255.0 - ): - if p_len is None: p_len = x.shape[0] // self.window - compute_fn = self.get_f0_hybrid if "hybrid" in f0_method else self.compute_f0 - - f0 = compute_fn( - f0_method, - x, - p_len, - filter_radius if filter_radius % 2 != 0 else filter_radius + 1 - ) - - if proposal_pitch: - up_key = proposal_f0_up_key( - f0, - proposal_pitch_threshold, - configs["limit_f0"] - ) - - logger.debug(translations["proposal_f0"].format(up_key=up_key)) - f0_up_key += up_key - - if f0_autotune: - logger.debug(translations["startautotune"]) - - f0 = autotune_f0( - self.ref_freqs, - f0, - f0_autotune_strength - ) - - return post_process(f0, f0_up_key, x_pad, self.f0_mel_min, self.f0_mel_max, manual_f0) - - def realtime_calculator( - self, - audio, - f0_method, - pitch, - pitchf, - f0_up_key = 0, - filter_radius = 3, - f0_autotune = False, - f0_autotune_strength = 1, - proposal_pitch = False, - proposal_pitch_threshold = 255.0 - ): - p_len = audio.shape[0] // self.window - - f0 = self.compute_f0( - f0_method, - audio, - p_len, - filter_radius if filter_radius % 2 != 0 else filter_radius + 1 - ) - - if f0_autotune: - f0 = autotune_f0( - self.ref_freqs, - f0, - f0_autotune_strength - ) - - if proposal_pitch: - up_key = proposal_f0_up_key( - f0, - proposal_pitch_threshold, - configs["limit_f0"] - ) - - f0_up_key += up_key - - return realtime_post_process(f0, pitch, pitchf, f0_up_key, self.f0_min, self.f0_max) - - def _resize_array_f0(self, x, target_len, resize = False): - if torch.is_tensor(x): x = x.cpu().numpy().astype(np.float32) - if not resize or len(x) == target_len: return x - - source = np.array(x) - source[source < 0.001] = np.nan - - return np.nan_to_num( - np.interp( - np.arange(0, len(source) * target_len, len(source)) / target_len, - np.arange(0, len(source)), - source - ) - ) - - def _resize_tensor_f0(self, x, target_len, resize = False): - if not torch.is_tensor(x): x = torch.from_numpy(x).to(self.device) - if not resize or len(x) == target_len: return x - - source = torch.as_tensor(x, dtype=self.resize_dtype, device=self.device).clone() - source[source < 0.001] = torch.nan - n = source.shape[0] - - xp = torch.arange(target_len, dtype=self.resize_dtype, device=self.device) * (n / target_len) - x0 = xp.floor().long() - - y0 = source[x0] - y1 = source[(x0 + 1).clamp(max=n - 1)] - w = xp - x0.to(self.resize_dtype) - - return torch.nan_to_num( - torch.where( - w == 0, - y0, - y0 + (y1 - y0) * w - ) - ) - - def compute_f0(self, f0_method, x, p_len, filter_radius): - if f0_method != self.cache_f0_method: - self.cache_f0_method = f0_method - self.cache_args = f0_method.split("-") - self.cache_flags = set(self.cache_args) - - if "pm" in self.cache_flags: - f0 = self.get_f0_pm( - x, - p_len, - filter_radius=filter_radius, - mode=self.cache_args[1] - ) - elif self.cache_args[0] in {"harvest", "dio"}: - f0 = self.get_f0_pyworld( - x, - p_len, - filter_radius, - self.cache_args[0], - use_stonemask="stonemask" in self.cache_flags - ) - elif "crepe" in self.cache_flags: - f0 = ( - self.get_f0_mangio_crepe( - x, - p_len, - self.cache_args[2] - ) - ) if "mangio" in self.cache_flags else ( - self.get_f0_crepe( - x, - p_len, - self.cache_args[1], - filter_radius=filter_radius - ) - ) - elif "fcpe" in self.cache_flags: - f0 = self.get_f0_fcpe( - x, - p_len, - legacy="legacy" in self.cache_flags and "previous" not in self.cache_flags, - previous="previous" in self.cache_flags, - filter_radius=filter_radius - ) - elif "rmvpe" in self.cache_flags: - f0 = self.get_f0_rmvpe( - x, - p_len, - clipping="clipping" in self.cache_flags, - filter_radius=filter_radius, - hpa="hpa" in self.cache_flags, - previous="previous" in self.cache_flags, - mix="mix" in self.cache_flags - ) - elif self.cache_args[0] in {"yin", "pyin", "piptrack"}: - f0 = self.get_f0_librosa( - x, - p_len, - mode=self.cache_args[0], - filter_radius=filter_radius - ) - elif "swipe" in self.cache_flags: - f0 = self.get_f0_swipe( - x, - p_len, - filter_radius=filter_radius, - use_stonemask="stonemask" in self.cache_flags - ) - elif "penn" in self.cache_flags: - f0 = ( - self.get_f0_mangio_penn( - x, - p_len - ) - ) if self.cache_args[0] == "mangio" else ( - self.get_f0_penn( - x, - p_len, - filter_radius=filter_radius - ) - ) - elif "djcm" in self.cache_flags: - f0 = self.get_f0_djcm( - x, - p_len, - clipping="clipping" in self.cache_flags, - svs="svs" in self.cache_flags, - filter_radius=filter_radius - ) - elif "pesto" in self.cache_flags: - f0 = self.get_f0_pesto( - x, - p_len - ) - elif "swift" in self.cache_flags: - f0 = self.get_f0_swift( - x, - p_len, - filter_radius=filter_radius - ) - else: - raise ValueError(translations["option_not_valid"]) - - if isinstance(f0, tuple): f0 = f0[0] - if "medfilt" in self.cache_flags or "svs" in self.cache_flags: f0 = medfilts(f0, kernel_size=5) - - return f0 - - def get_f0_hybrid(self, methods_str, x, p_len, filter_radius): - logger.debug(translations["hybrid_calc"].format(f0_method=methods_str)) - - methods_str = re.search(r"hybrid\[(.+)\]", methods_str) - if methods_str: methods = [method.strip() for method in methods_str.group(1).split("+")] - - n = len(methods) - f0_stack = [] - - for method in methods: - f0_stack.append( - self._resize_f0( - self.compute_f0( - method, - x, - p_len, - filter_radius - ), - p_len, - True - ) - ) - - f0_mix = np.zeros(p_len) - - if not f0_stack: return f0_mix - if len(f0_stack) == 1: return f0_stack[0] - - weights = (1 - np.abs(np.arange(n) / (n - 1) - (1 - self.alpha))) ** 2 - weights /= weights.sum() - - stacked = np.vstack(f0_stack) - voiced_mask = np.any(stacked > 0, axis=0) - - f0_mix[voiced_mask] = np.exp( - np.nansum( - np.log(stacked + 1e-6) * weights[:, None], axis=0 - )[voiced_mask] - ) - - return f0_mix - - def get_f0_pm(self, x, p_len, filter_radius=3, mode="ac"): - if torch.is_tensor(x): x = x.cpu().numpy() - - pm = parselmouth.Sound(x, self.sample_rate) - - pm_fn = { - "ac": pm.to_pitch_ac, - "cc": pm.to_pitch_cc, - "shs": pm.to_pitch_shs - }.get(mode, pm.to_pitch_ac) - - pitch = ( - pm_fn( - time_step=self.time_step, - voicing_threshold=filter_radius / 10 * 2, - pitch_floor=self.f0_min, - pitch_ceiling=self.f0_max - ) - ) if mode != "shs" else ( - pm_fn( - time_step=self.time_step, - minimum_pitch=self.f0_min, - maximum_frequency_component=self.f0_max - ) - ) - - f0 = pitch.selected_array["frequency"] - pad_size = (p_len - len(f0) + 1) // 2 - - if pad_size > 0 or p_len - len(f0) - pad_size > 0: f0 = np.pad(f0, [[pad_size, p_len - len(f0) - pad_size]], mode="constant") - return self._resize_f0(f0, p_len, self.resize) - - def get_f0_mangio_crepe(self, x, p_len, model="full"): - if not hasattr(self, "mangio_crepe"): - from main.library.predictors.CREPE.CREPE import CREPE - - self.mangio_crepe = CREPE( - os.path.join( - configs["predictors_path"], - f"crepe_{model}.{'onnx' if self.predictor_onnx else 'pth'}" - ), - model_size=model, - hop_length=self.hop_length, - batch_size=self.hop_length * 2, - f0_min=self.f0_min, - f0_max=self.f0_max, - device=self.device, - sample_rate=self.sample_rate, - providers=self.providers, - onnx=self.predictor_onnx, - return_periodicity=False, - compile_model=self.compile_model, - compile_mode=self.compile_mode - ) - - if not torch.is_tensor(x): x = torch.from_numpy(x).to(device=self.device) - - x = x.float() - x /= x.abs().quantile(0.999) - audio = x.unsqueeze(dim=0) - - if audio.ndim == 2 and audio.shape[0] > 1: audio = audio.mean(dim=0, keepdim=True).detach() - f0 = self.mangio_crepe.compute_f0(audio.detach(), pad=True).squeeze(0) - - return self._resize_f0(f0, p_len, True) - - def get_f0_crepe(self, x, p_len, model="full", filter_radius=3): - if not hasattr(self, "crepe"): - from main.library.predictors.CREPE.CREPE import CREPE - - self.crepe = CREPE( - os.path.join( - configs["predictors_path"], - f"crepe_{model}.{'onnx' if self.predictor_onnx else 'pth'}" - ), - model_size=model, - hop_length=self.window, - batch_size=self.batch_size, - f0_min=self.f0_min, - f0_max=self.f0_max, - device=self.device, - sample_rate=self.sample_rate, - providers=self.providers, - onnx=self.predictor_onnx, - return_periodicity=True, - compile_model=self.compile_model, - compile_mode=self.compile_mode - ) - - if not torch.is_tensor(x): x = torch.from_numpy(x).to(device=self.device) - - f0, pd = self.crepe.compute_f0(x.float().unsqueeze(0), pad=True) - f0, pd = mean(f0, filter_radius), median(pd, filter_radius) - f0[pd < 0.1] = 0 - - return self._resize_f0(f0[0], p_len, self.resize) - - def get_f0_fcpe(self, x, p_len, legacy=False, previous=False, filter_radius=3): - if not hasattr(self, "fcpe"): - from main.library.predictors.FCPE.FCPE import FCPE - - self.fcpe = FCPE( - os.path.join( - configs["predictors_path"], - ( - "fcpe_legacy" - if legacy else - ("fcpe" if previous else "ddsp_200k") - ) + (".onnx" if self.predictor_onnx else ".pt") - ), - device=self.device, - threshold=(filter_radius / 100) if legacy else (filter_radius / 1000 * 2), - providers=self.providers, - onnx=self.predictor_onnx, - legacy=legacy, - compile_model=self.compile_model, - compile_mode=self.compile_mode - ) - - f0 = self.fcpe.compute_f0(x) - return self._resize_f0(f0, p_len, self.resize) - - def get_f0_rmvpe(self, x, p_len, clipping=False, filter_radius=3, hpa=False, previous=False, mix=False): - if not hasattr(self, "rmvpe"): - from main.library.predictors.RMVPE.RMVPE import RMVPE - - self.rmvpe = RMVPE( - os.path.join( - configs["predictors_path"], - ( - ( - "hpa-rmvpe-76000" - if previous else - "hpa-rmvpe-112000" - ) if hpa else ("rmvpe-mix" if mix else "rmvpe") - ) + (".onnx" if self.predictor_onnx else ".pt") - ), - is_half=self.is_half, - device=self.device, - onnx=self.predictor_onnx, - providers=self.providers, - hpa=hpa, - compile_model=self.compile_model, - compile_mode=self.compile_mode, - enable_chunk=configs.get("enable_rmvpe_chunk", False) or self.device.startswith(("privateuseone", "ocl")), - chunk_size=configs.get("rmvpe_chunk_size", 8000), - return_tensor=self.return_tensor, - f0_min=self.f0_min, - f0_max=self.f0_max - ) - if clipping: self.rmvpe.infer_from_audio = self.rmvpe.infer_from_audio_with_pitch - - f0 = self.rmvpe.infer_from_audio(x, thred=filter_radius / 100) - return self._resize_f0(f0, p_len, self.resize) - - def get_f0_pyworld(self, x, p_len, filter_radius, model="harvest", use_stonemask=True): - if not hasattr(self, "pw"): - from main.library.predictors.WORLD.WORLD import PYWORLD - - self.pw = PYWORLD( - os.path.join(configs["predictors_path"], "world"), - os.path.join(configs["binary_path"], "world.bin"), - harvest=model == "harvest" - ) - - if torch.is_tensor(x): x = x.cpu().numpy() - x = x.astype(np.double) - - f0, t = self.pw.infer( - x, - fs=self.sample_rate, - f0_ceil=self.f0_max, - f0_floor=self.f0_min, - frame_period=self.frame_period - ) - - if use_stonemask: - f0 = self.pw.stonemask( - x, - self.sample_rate, - t, - f0 - ) - - if filter_radius > 2 and model == "harvest": f0 = medfilts(f0, filter_radius) - elif model == "dio": - for index, pitch in enumerate(f0): - f0[index] = round(pitch, 1) - - return self._resize_f0(f0, p_len, self.resize) - - def get_f0_swipe(self, x, p_len, filter_radius=3, use_stonemask=True): - if torch.is_tensor(x): x = x.cpu().numpy() - - f0, t = swipe( - x.astype(np.float32), - self.sample_rate, - f0_floor=self.f0_min, - f0_ceil=self.f0_max, - frame_period=self.frame_period, - sTHR=filter_radius / 10 - ) - - if use_stonemask: - f0 = stonemask( - x, - self.sample_rate, - t, - f0 - ) - - return self._resize_f0(f0, p_len, self.resize) - - def get_f0_librosa(self, x, p_len, mode="yin", filter_radius=3): - if torch.is_tensor(x): x = x.cpu().numpy() - - if mode != "piptrack": - if_yin = mode == "yin" - yin_fn = yin if if_yin else pyin - - f0 = yin_fn( - x.astype(np.float32), - sr=self.sample_rate, - fmin=self.f0_min, - fmax=self.f0_max, - hop_length=self.hop_length - ) - - if not if_yin: f0 = f0[0] - else: - pitches, magnitudes = piptrack( - y=x.astype(np.float32), - sr=self.sample_rate, - fmin=self.f0_min, - fmax=self.f0_max, - hop_length=self.hop_length, - threshold=filter_radius / 10 - ) - - f0 = pitches[np.argmax(magnitudes, axis=0), range(magnitudes.shape[1])] - - return self._resize_f0(f0, p_len, True) - - def get_f0_penn(self, x, p_len, filter_radius=3): - if not hasattr(self, "penn"): - from main.library.predictors.PENN.PENN import PENN - - self.penn = PENN( - os.path.join( - configs["predictors_path"], - f"fcn.{'onnx' if self.predictor_onnx else 'pt'}" - ), - hop_length=self.window, - batch_size=self.batch_size, - f0_min=self.f0_min, - f0_max=self.f0_max, - device=self.device, - providers=self.providers, - onnx=self.predictor_onnx, - compile_model=self.compile_model, - compile_mode=self.compile_mode - ) - - if not torch.is_tensor(x): x = torch.from_numpy(x).to(device=self.device) - - f0, pd = self.penn.compute_f0(x.float().unsqueeze(0)) - f0, pd = mean(f0, filter_radius), median(pd, filter_radius) - f0[pd < 0.1] = 0 - - f0 = medfilts(f0[0], kernel_size=5) - return self._resize_f0(f0, p_len, True) - - def get_f0_mangio_penn(self, x, p_len): - if not hasattr(self, "mangio_penn"): - from main.library.predictors.PENN.PENN import PENN - - self.mangio_penn = PENN( - os.path.join( - configs["predictors_path"], - f"fcn.{'onnx' if self.predictor_onnx else 'pt'}" - ), - hop_length=self.hop_length, - batch_size=self.hop_length * 2, - f0_min=self.f0_min, - f0_max=self.f0_max, - device=self.device, - providers=self.providers, - onnx=self.predictor_onnx, - interp_unvoiced_at=0.1, - compile_model=self.compile_model, - compile_mode=self.compile_mode - ) - - if not torch.is_tensor(x): x = torch.from_numpy(x).to(device=self.device) - - x = x.float() - x /= x.abs().quantile(0.999) - audio = x.unsqueeze(dim=0) - - if audio.ndim == 2 and audio.shape[0] > 1: audio = audio.mean(dim=0, keepdim=True).detach() - f0 = self.mangio_penn.compute_f0(audio.detach()).squeeze(0) - - f0 = medfilts(f0, kernel_size=5) - return self._resize_f0(f0, p_len, True) - - def get_f0_djcm(self, x, p_len, clipping=False, svs=False, filter_radius=3): - if not hasattr(self, "djcm"): - from main.library.predictors.DJCM.DJCM import DJCM - - self.djcm = DJCM( - os.path.join( - configs["predictors_path"], - ( - "djcm-svs" - if svs else - "djcm" - ) + (".onnx" if self.predictor_onnx else ".pt") - ), - is_half=self.is_half, - device=self.device, - onnx=self.predictor_onnx, - svs=svs, - providers=self.providers, - compile_model=self.compile_model, - compile_mode=self.compile_mode, - return_tensor=self.return_tensor, - f0_min=self.f0_min, - f0_max=self.f0_max - ) - if clipping: self.djcm.infer_from_audio = self.djcm.infer_from_audio_with_pitch - - f0 = self.djcm.infer_from_audio(x, thred=filter_radius / 10) - return self._resize_f0(f0, p_len, self.resize) - - def get_f0_swift(self, x, p_len, filter_radius=3): - if not hasattr(self, "swift"): - from main.library.predictors.SWIFT.SWIFT import SWIFT - - self.swift = SWIFT( - os.path.join( - configs["predictors_path"], - "swift.onnx" - ), - fmin=self.f0_min, - fmax=self.f0_max, - confidence_threshold=filter_radius / 4 + 0.137 - ) - - if torch.is_tensor(x): x = x.cpu().numpy() - - pitch_hz, _, _ = self.swift.detect_from_array(x) - return self._resize_f0(pitch_hz, p_len, self.resize) - - def get_f0_pesto(self, x, p_len): - if not hasattr(self, "pesto"): - from main.library.predictors.PESTO.PESTO import PESTO - - self.pesto = PESTO( - os.path.join( - configs["predictors_path"], - f"pesto.{'onnx' if self.predictor_onnx else 'pt'}" - ), - step_size=self.frame_period, - reduction = "alwa", - sample_rate=self.sample_rate, - device=self.device, - providers=self.providers, - onnx=self.predictor_onnx, - compile_model=self.compile_model, - compile_mode=self.compile_mode, - chunk_size=30 * self.sample_rate - ) - - if not torch.is_tensor(x): x = torch.from_numpy(x).to(device=self.device) - - x = x.float() - x /= x.abs().quantile(0.999) - audio = x.unsqueeze(dim=0) - - if audio.ndim == 2 and audio.shape[0] > 1: audio = audio.mean(dim=0, keepdim=True).detach() - f0 = self.pesto.compute_f0(audio.detach())[0].squeeze(0) - - return self._resize_f0(f0, p_len, self.resize) \ No newline at end of file diff --git a/main/library/predictors/PENN/PENN.py b/main/library/predictors/PENN/PENN.py deleted file mode 100644 index 4b7458433e7a3e6f2cf785705c3495ca757905d0..0000000000000000000000000000000000000000 --- a/main/library/predictors/PENN/PENN.py +++ /dev/null @@ -1,141 +0,0 @@ -import os -import sys -import torch - -import torch.nn.functional as F - -sys.path.append(os.getcwd()) - -from main.library.algorithm.viterbi import viterbi -from main.library.predictors.PENN.core import bins_to_cents, cents_to_frequency -from main.library.predictors.PENN.core import PITCH_BINS, CENTS_PER_BIN, frequency_to_bins, entropy, interpolate - -SAMPLE_RATE, WINDOW_SIZE = 16000, 1024 - -class Viterbi: - def __init__( - self, - pitch_bins=1440, - hop_length=80, - sample_rate=16000, - local_pitch_window_size=19, - max_octaves_per_second=32, - cents_per_bin=5 - ): - self.pitch_bins = pitch_bins - self.hop_length = hop_length - self.sample_rate = sample_rate - self.window_size = local_pitch_window_size - self.max_octaves_per_second = max_octaves_per_second - self.cents_per_bin = cents_per_bin - - def __call__(self, logits): - if not hasattr(self, 'transition'): - idx = torch.arange(self.pitch_bins, device=logits.device, dtype=logits.dtype) - transition = (-0.5 * ((idx[:, None] - idx[None, :]).abs() / ((self.max_octaves_per_second * 1200 * (self.hop_length / self.sample_rate)) / self.cents_per_bin)) ** 2).exp() - self.transition = transition / transition.sum(dim=1, keepdim=True) - - distributions = F.softmax(logits, dim=1) - bins = viterbi(distributions, self.transition) - - pitch = self.local_expected_value_from_bins(bins.T, logits) - return pitch - - def local_expected_value_from_bins(self, bins, logits): - batch, pitch_bins, frames = logits.shape - logits_flat = logits.reshape(-1, pitch_bins) - bins_flat = bins.reshape(-1) - - half_win = self.window_size // 2 - steps = torch.arange(-half_win, half_win + 1, device=bins.device) - indices = (bins_flat[:, None] + steps[None, :]).clamp(0, pitch_bins - 1) - - return cents_to_frequency((F.softmax(logits_flat.gather(1, indices), dim=1) * bins_to_cents(indices)).sum(dim=1, keepdim=True)).reshape(batch, frames) - -class PENN: - def __init__( - self, - model_path, - hop_length = 80, - batch_size = None, - f0_min = 31, - f0_max = 1984, - interp_unvoiced_at = None, - device = None, - providers = None, - onnx = False, - compile_model = False, - compile_mode = None - ): - self.device = device - self.hop_length = hop_length - self.batch_size = batch_size or 384 - self.f0_min_bin = frequency_to_bins(torch.tensor(f0_min)) - self.f0_max_bin = frequency_to_bins(torch.tensor(f0_max), torch.ceil) - self.interp_unvoiced_at = interp_unvoiced_at - self.decoder = Viterbi( - PITCH_BINS, - hop_length, - SAMPLE_RATE, - 19, - 32, - CENTS_PER_BIN - ) - - if onnx: - import onnxruntime as ort - - sess_options = ort.SessionOptions() - sess_options.log_severity_level = 3 - model = ort.InferenceSession(model_path, sess_options=sess_options, providers=providers) - else: - from main.library.predictors.PENN.fcn import FCN - - model = FCN(256, PITCH_BINS, (2, 2)) - model.load_state_dict(torch.load(model_path, map_location="cpu", weights_only=True)['model']) - model.to(device).eval() - if compile_model: model = torch.compile(model, mode=compile_mode) - - self.model = model - self.infer = self._infer_onnx if onnx else self._infer_torch - - def preprocess(self, audio): - audio = F.pad(audio.to(self.device), (WINDOW_SIZE // 2, WINDOW_SIZE // 2)) - - frames = audio.unfold(-1, WINDOW_SIZE, self.hop_length).squeeze(0) - frames = frames.unsqueeze(1).contiguous() - - return frames - - def postprocess(self, logits): - with torch.inference_mode(): - logits[:, :self.f0_min_bin] = -float('inf') - logits[:, self.f0_max_bin:] = -float('inf') - - pitch = self.decoder(logits) - periodicity = entropy(logits) - - return pitch.T, periodicity.T - - def compute_f0(self, audio): - with torch.inference_mode(): - frames = self.preprocess(audio) - pitch, periodicity = self.postprocess(torch.cat([self.infer(frames[i:i + self.batch_size]).detach() for i in range(0, frames.shape[0], self.batch_size)], dim=0)) - - if self.interp_unvoiced_at is not None: - pitch = interpolate(pitch, periodicity, self.interp_unvoiced_at) - return pitch - - return pitch, periodicity - - def _infer_onnx(self, frames): - return torch.tensor( - self.model.run( - [self.model.get_outputs()[0].name], - {self.model.get_inputs()[0].name: frames.cpu().numpy()} - )[0], - device=self.device - ) - - def _infer_torch(self, frames): - return self.model(frames) \ No newline at end of file diff --git a/main/library/predictors/PENN/core.py b/main/library/predictors/PENN/core.py deleted file mode 100644 index eb632bf3cf179b82bc45edd349dbeeb10957c8d1..0000000000000000000000000000000000000000 --- a/main/library/predictors/PENN/core.py +++ /dev/null @@ -1,77 +0,0 @@ -import math -import torch - -import torch.nn.functional as F - -PITCH_BINS, CENTS_PER_BIN, OCTAVE = 1440, 5, 1200 - -def frequency_to_bins(frequency, quantize_fn=torch.floor): - return cents_to_bins(frequency_to_cents(frequency), quantize_fn) - -def cents_to_bins(cents, quantize_fn=torch.floor): - bins = quantize_fn(cents / CENTS_PER_BIN).long() - bins[bins < 0] = 0 - bins[bins >= PITCH_BINS] = PITCH_BINS - 1 - return bins - -def cents_to_frequency(cents): - return 31 * 2 ** (cents / OCTAVE) - -def bins_to_cents(bins): - return CENTS_PER_BIN * bins - -def frequency_to_cents(frequency): - return OCTAVE * (frequency / 31).log2() - -def interpolate(pitch, periodicity, value): - voiced = periodicity > value - if not voiced.any(): return pitch - - pitch = pitch.log2() - pitch[..., 0] = pitch[voiced][..., 0] - pitch[..., -1] = pitch[voiced][..., -1] - voiced[..., 0] = True - voiced[..., -1] = True - - pitch[~voiced] = _interpolate( - torch.where(~voiced[0])[0][None], - torch.where(voiced[0])[0][None], - pitch[voiced][None] - ) - - return 2 ** pitch - -def _interpolate(x, xp, fp): - if xp.shape[-1] == 0: return x - - if xp.shape[-1] == 1: - return torch.full( - x.shape, - fp.squeeze(), - device=fp.device, - dtype=fp.dtype - ) - - m = (fp[:, 1:] - fp[:, :-1]) / (xp[:, 1:] - xp[:, :-1]) - b = fp[:, :-1] - (m.mul(xp[:, :-1])) - - indicies = x[:, :, None].ge(xp[:, None, :]).sum(-1) - 1 - indicies = indicies.clamp(0, m.shape[-1] - 1) - - line_idx = torch.linspace( - 0, - indicies.shape[0], - 1, - device=indicies.device - ).to(torch.long).expand(indicies.shape) - - return m[line_idx, indicies].mul(x) + b[line_idx, indicies] - -def entropy(logits): - distribution = F.softmax(logits, dim=1) - - return ( - 1 + 1 / math.log(PITCH_BINS) * ( - distribution * (distribution + 1e-7).log() - ).sum(dim=1) - ) \ No newline at end of file diff --git a/main/library/predictors/PENN/fcn.py b/main/library/predictors/PENN/fcn.py deleted file mode 100644 index 04f05ddcfd3121bb82c2419370911197dd9a9483..0000000000000000000000000000000000000000 --- a/main/library/predictors/PENN/fcn.py +++ /dev/null @@ -1,83 +0,0 @@ -import torch - -class FCN(torch.nn.Sequential): - def __init__( - self, - channels = 256, - pitch_bins = 1440, - pooling = (2, 2) - ): - super().__init__(*( - Block( - 1, - channels, - 481, - pooling - ), - Block( - channels, - channels // 8, - 225, - pooling - ), - Block( - channels // 8, - channels // 8, - 97, - pooling - ), - Block( - channels // 8, - channels // 2, - 66 - ), - Block( - channels // 2, - channels, - 35 - ), - Block( - channels, - channels * 2, - 4 - ), - torch.nn.Conv1d( - channels * 2, - pitch_bins, - 4 - ) - )) - - def forward(self, frames): - return super().forward( - frames[:, :, 16:-15] - ) - -class Block(torch.nn.Sequential): - def __init__( - self, - in_channels, - out_channels, - length=1, - pooling=None, - kernel_size=32 - ): - layers = ( - torch.nn.Conv1d( - in_channels, - out_channels, - kernel_size - ), - torch.nn.ReLU() - ) - - if pooling is not None: - layers += ( - torch.nn.MaxPool1d(*pooling), - ) - - layers += ( - torch.nn.LayerNorm((out_channels, length)), - ) - - super().__init__(*layers) \ No newline at end of file diff --git a/main/library/predictors/PESTO/PESTO.py b/main/library/predictors/PESTO/PESTO.py deleted file mode 100644 index 1ede65412286e609b36cb982bda06c455684d2cc..0000000000000000000000000000000000000000 --- a/main/library/predictors/PESTO/PESTO.py +++ /dev/null @@ -1,85 +0,0 @@ -import os -import sys -import torch - -sys.path.append(os.getcwd()) - -class PESTO: - def __init__( - self, - model_path, - step_size=10, - reduction="alwa", - sample_rate=16000, - device=None, - providers=None, - onnx=False, - compile_model=False, - compile_mode=None, - chunk_size = None - ): - self.device = device - self.step_size = step_size - self.sample_rate = sample_rate - self.chunk_size = chunk_size - - if onnx: - import onnxruntime as ort - - sess_options = ort.SessionOptions() - sess_options.log_severity_level = 3 - model = ort.InferenceSession(model_path, sess_options=sess_options, providers=providers) - else: - from main.library.predictors.PESTO.model import PPESTO, Resnet1d - from main.library.predictors.PESTO.preprocessor import Preprocessor - - ckpt = torch.load(model_path, map_location="cpu", weights_only=True) - model = PPESTO( - Resnet1d( - **ckpt["hparams"]["encoder"] - ), - preprocessor=Preprocessor( - hop_size=step_size, - sampling_rate=sample_rate, - **ckpt["hcqt_params"] - ), - crop_kwargs=ckpt["hparams"]["pitch_shift"], - reduction=reduction or ckpt["hparams"]["reduction"] - ) - model.load_state_dict(ckpt["state_dict"], strict=False) - model.to(self.device).eval() - if compile_model: model = torch.compile(model, mode=compile_mode) - - self.model = model - self.infer = self._infer_onnx if onnx else self._infer_torch - - def compute_f0(self, x): - with torch.inference_mode(): - assert x.ndim <= 2 - - preds, confidence = [], [] - total_samples = x.shape[-1] - if total_samples <= self.chunk_size: return self.infer(x) - - for i in range(0, total_samples, self.chunk_size): - pred_chunk, conf_chunk = self.infer(x[i : i + self.chunk_size] if x.ndim == 1 else x[:, i : i + self.chunk_size]) - - preds.append(pred_chunk) - confidence.append(conf_chunk) - - return torch.cat(preds, dim=-1), torch.cat(confidence, dim=-1) - - def _infer_onnx(self, x): - model = self.model.run( - [self.model.get_outputs()[0].name, self.model.get_outputs()[1].name], - {self.model.get_inputs()[0].name: x.cpu().numpy()} - ) - return torch.tensor(model[0], device=self.device), torch.tensor(model[1], device=self.device) - - def _infer_torch(self, x): - return self.model( - x, - sr=self.sample_rate, - convert_to_freq=True, - return_activations=False - ) \ No newline at end of file diff --git a/main/library/predictors/PESTO/model.py b/main/library/predictors/PESTO/model.py deleted file mode 100644 index be171f5d02350ebf0c613a200df6c41ce943be97..0000000000000000000000000000000000000000 --- a/main/library/predictors/PESTO/model.py +++ /dev/null @@ -1,224 +0,0 @@ -import math -import torch - -from functools import partial - -class PPESTO(torch.nn.Module): - def __init__( - self, - encoder, - preprocessor, - crop_kwargs = None, - reduction = "alwa" - ): - super(PPESTO, self).__init__() - self.encoder = encoder - self.preprocessor = preprocessor - self.confidence = ConfidenceClassifier() - if crop_kwargs is None: crop_kwargs = {} - self.crop_cqt = CropCQT(**crop_kwargs) - self.reduction = reduction - self.register_buffer('shift', torch.zeros((), dtype=torch.float), persistent=True) - - def forward(self, audio_waveforms, sr = 16000, convert_to_freq = True, return_activations = False): - batch_size = audio_waveforms.size(0) if audio_waveforms.ndim == 2 else None - x = self.preprocessor(audio_waveforms, sr=sr).flatten(0, 1) - - energy = x.mul_(math.log(10) / 10.).exp().squeeze_(1) - vol = energy.sum(dim=-1) - confidence = self.confidence(energy) - - x = self.crop_cqt(x) - activations = self.encoder(x) - - if batch_size is None: confidence.squeeze_(0) - else: - activations = activations.view(batch_size, -1, activations.size(-1)) - confidence = confidence.view(batch_size, -1) - vol = vol.view(batch_size, -1) - - activations = activations.roll(-(self.shift * self.preprocessor.hcqt_kwargs["bins_per_semitone"]).round().int().item(), -1) - preds = self.reduce_activations(activations.cpu()).to(activations.device) if activations.device.type.startswith(("ocl", "privateuseone")) else self.reduce_activations(activations) - if convert_to_freq: preds = 440 * 2 ** ((preds - 69) / 12) - - if return_activations: return preds, confidence, vol, activations - return preds, confidence - - def reduce_activations(self, activations): - device = activations.device - num_bins = activations.size(-1) - if torch.is_tensor(num_bins): num_bins = num_bins.item() - - bps, r = divmod(num_bins, 128) - assert r == 0 - - if self.reduction == "argmax": return activations.argmax(dim=-1).float() / bps - - all_pitches = torch.arange(num_bins, dtype=torch.float, device=device).div_(bps) - if self.reduction == "mean": return activations.matmul(all_pitches) - - if self.reduction == "alwa": - indices = ( - activations.argmax(dim=-1, keepdim=True) + - (torch.arange(1, 2 * bps, device=device) - bps) - ).clip_(min=0, max=num_bins - 1) - cropped_activations = activations.gather(-1, indices) - - return ( - cropped_activations * all_pitches.unsqueeze(0).expand_as(activations).gather(-1, indices) - ).sum(dim=-1) / cropped_activations.sum(dim=-1) - - raise ValueError - -class ConfidenceClassifier(torch.nn.Module): - def __init__(self): - super(ConfidenceClassifier, self).__init__() - self.conv = torch.nn.Conv1d(1, 1, 39, stride=3) - self.linear = torch.nn.Linear(72, 1) - - def forward(self, x): - return self.linear( - torch.cat( - ( - torch.nn.functional.relu( - self.conv(x.unsqueeze(1)).squeeze(1) - ), - x.log().mean(dim=-1, keepdim=True).exp() / x.mean(dim=-1, keepdim=True).clip_(min=1e-8) - ), - dim=-1 - ) - ).sigmoid().squeeze(-1) - -class CropCQT(torch.nn.Module): - def __init__(self, min_steps, max_steps): - super(CropCQT, self).__init__() - self.min_steps = min_steps - self.max_steps = max_steps - self.lower_bin = self.max_steps - - def forward(self, spectrograms): - return spectrograms[..., self.max_steps: self.min_steps] - -class Resnet1d(torch.nn.Module): - def __init__( - self, - n_chan_input=1, - n_chan_layers=(20, 20, 10, 1), - n_prefilt_layers=1, - prefilt_kernel_size=15, - residual=False, - n_bins_in=216, - output_dim=128, - activation_fn = "leaky", - a_lrelu=0.3, - p_dropout=0.2, - **unused - ): - super(Resnet1d, self).__init__() - self.hparams = dict( - n_chan_input=n_chan_input, - n_chan_layers=n_chan_layers, - n_prefilt_layers=n_prefilt_layers, - prefilt_kernel_size=prefilt_kernel_size, - residual=residual, - n_bins_in=n_bins_in, - output_dim=output_dim, - activation_fn=activation_fn, - a_lrelu=a_lrelu, - p_dropout=p_dropout - ) - - if activation_fn == "relu": - activation_layer = torch.nn.ReLU - elif activation_fn == "silu": - activation_layer = torch.nn.SiLU - elif activation_fn == "leaky": - activation_layer = partial(torch.nn.LeakyReLU, negative_slope=a_lrelu) - else: - raise ValueError - - n_in = n_chan_input - n_ch = n_chan_layers - if len(n_ch) < 5: n_ch.append(1) - - self.layernorm = torch.nn.LayerNorm(normalized_shape=[n_in, n_bins_in]) - prefilt_padding = prefilt_kernel_size // 2 - - self.conv1 = torch.nn.Sequential( - torch.nn.Conv1d( - in_channels=n_in, - out_channels=n_ch[0], - kernel_size=prefilt_kernel_size, - padding=prefilt_padding, - stride=1 - ), - activation_layer(), - torch.nn.Dropout(p=p_dropout) - ) - - self.n_prefilt_layers = n_prefilt_layers - self.prefilt_layers = torch.nn.ModuleList([ - torch.nn.Sequential( - torch.nn.Conv1d( - in_channels=n_ch[0], - out_channels=n_ch[0], - kernel_size=prefilt_kernel_size, - padding=prefilt_padding, - stride=1 - ), - activation_layer(), - torch.nn.Dropout(p=p_dropout) - ) - for _ in range(n_prefilt_layers-1) - ]) - - self.residual = residual - conv_layers = [] - - for i in range(len(n_chan_layers)-1): - conv_layers.extend([ - torch.nn.Conv1d( - in_channels=n_ch[i], - out_channels=n_ch[i + 1], - kernel_size=1, - padding=0, - stride=1 - ), - activation_layer(), - torch.nn.Dropout(p=p_dropout) - ]) - - self.conv_layers = torch.nn.Sequential(*conv_layers) - self.flatten = torch.nn.Flatten(start_dim=1) - self.fc = ToeplitzLinear(n_bins_in * n_ch[-1], output_dim) - self.final_norm = torch.nn.Softmax(dim=-1) - - def forward(self, x): - x = self.conv1(self.layernorm(x)) - - for p in range(0, self.n_prefilt_layers - 1): - prefilt_layer = self.prefilt_layers[p] - - if self.residual: - x = prefilt_layer(x) + x - else: - x = prefilt_layer(x) - - return self.final_norm(self.fc(self.flatten(self.conv_layers(x)))) - -class ToeplitzLinear(torch.nn.Conv1d): - def __init__( - self, - in_features, - out_features - ): - super(ToeplitzLinear, self).__init__( - in_channels=1, - out_channels=1, - kernel_size=in_features+out_features - 1, - padding=out_features - 1, - bias=False - ) - - def forward(self, input): - return super(ToeplitzLinear, self).forward(input.unsqueeze(-2)).squeeze(-2) \ No newline at end of file diff --git a/main/library/predictors/PESTO/preprocessor.py b/main/library/predictors/PESTO/preprocessor.py deleted file mode 100644 index 5b57c3dcb757e645e2499d60fc71066bdd3ee603..0000000000000000000000000000000000000000 --- a/main/library/predictors/PESTO/preprocessor.py +++ /dev/null @@ -1,339 +0,0 @@ -import torch - -import numpy as np - -from scipy.signal import get_window - -class Preprocessor(torch.nn.Module): - def __init__( - self, - hop_size, - sampling_rate = None, - **hcqt_kwargs - ): - super(Preprocessor, self).__init__() - self.hcqt_sr = None - self.hcqt_kernels = None - self.hop_size = hop_size - self.hcqt_kwargs = hcqt_kwargs - self.to_log = ToLogMagnitude() - self.register_buffer("_device", torch.zeros(()), persistent=False) - if sampling_rate is not None: - self.hcqt_sr = sampling_rate - self._reset_hcqt_kernels() - - def forward(self, x, sr = None): - return self.to_log(self.hcqt(x, sr=sr).permute(0, 3, 1, 2, 4)) - - def hcqt(self, audio, sr = None) : - if sr is not None and sr != self.hcqt_sr: - self.hcqt_sr = sr - self._reset_hcqt_kernels() - - return self.hcqt_kernels(audio) - - def _reset_hcqt_kernels(self): - self.hcqt_kernels = HarmonicCQT( - sr=self.hcqt_sr, - hop_length=int(self.hop_size * self.hcqt_sr / 1000 + 0.5), - **self.hcqt_kwargs - ).to(self._device.device) - -class ToLogMagnitude(torch.nn.Module): - def __init__(self): - super(ToLogMagnitude, self).__init__() - self.eps = torch.finfo(torch.float32).eps - - def forward(self, x): - x = (x[..., 0] ** 2 + x[..., 1] ** 2).sqrt() if x.shape[-1] == 2 else x.abs() - x.clamp_(min=self.eps).log10_().mul_(20) - - return x - -class HarmonicCQT(torch.nn.Module): - def __init__( - self, - harmonics, - sr = 22050, - hop_length = 512, - fmin = 32.7, - fmax = None, - bins_per_semitone = 1, - n_bins = 84, - center_bins = True, - gamma = 0, - center = True, - streaming = False, - mirror = 0, - max_batch_size = 1 - ): - super(HarmonicCQT, self).__init__() - if center_bins: fmin = fmin / 2 ** ((bins_per_semitone - 1) / (24 * bins_per_semitone)) - self.cqt_kernels = torch.nn.ModuleList([ - CQT( - sr=sr, - hop_length=hop_length, - fmin=h * fmin, - fmax=fmax, - n_bins=n_bins, - bins_per_octave=12 * bins_per_semitone, - gamma=gamma, - center=center, - streaming=streaming, - mirror=mirror, - max_batch_size=max_batch_size, - output_format="Complex" - ) - for h in harmonics - ]) - - def forward(self, audio_waveforms): - return torch.stack([cqt(audio_waveforms) for cqt in self.cqt_kernels], dim=1) - -class BaseCQT(torch.nn.Module): - def __init__( - self, - sr=22050, - hop_length=512, - fmin=32.70, - fmax=None, - n_bins=84, - bins_per_octave=12, - gamma=0, - filter_scale=1, - norm=1, - window="hann", - center = True, - trainable=False, - output_format="Magnitude" - ): - super(BaseCQT, self).__init__() - self.trainable = trainable - self.n_bins = n_bins - self.hop_length = hop_length - self.center = center - self.output_format = output_format - cqt_kernels, self.kernel_width, lengths, freqs = self.create_cqt_kernels( - float(filter_scale) / (2 ** (1 / bins_per_octave) - 1), - sr, - fmin, - n_bins, - bins_per_octave, - norm, window, - fmax, - gamma=gamma - ) - self.sqrt_lengths = lengths.sqrt_().unsqueeze_(-1) - self.frequencies = freqs - self.cqt_kernels = torch.from_numpy(cqt_kernels).unsqueeze(1) - - def create_cqt_kernels(self, Q, fs, fmin, n_bins=84, bins_per_octave=12, norm=1, window="hann", fmax=None, topbin_check=True, gamma=0): - fftLen = 2 ** int(np.ceil(np.log2(np.ceil(Q * fs / fmin)))) - - if (fmax != None) and (n_bins == None): - n_bins = np.ceil(bins_per_octave * np.log2(fmax / fmin)) - freqs = fmin * 2.0 ** (np.r_[0:n_bins] / np.double(bins_per_octave)) - elif (fmax == None) and (n_bins != None): freqs = fmin * 2.0 ** (np.r_[0:n_bins] / np.double(bins_per_octave)) - else: - n_bins = np.ceil(bins_per_octave * np.log2(fmax / fmin)) - freqs = fmin * 2.0 ** (np.r_[0:n_bins] / np.double(bins_per_octave)) - - if np.max(freqs) > fs / 2 and topbin_check == True: raise ValueError - - lengths = np.ceil(Q * fs / (freqs + gamma / (2.0 ** (1.0 / bins_per_octave) - 1.0))) - fftLen = int(2 ** (np.ceil(np.log2(int(max(lengths)))))) - tempKernel = np.zeros((int(n_bins), int(fftLen)), dtype=np.complex64) - - for k in range(0, int(n_bins)): - l = lengths[k] - start = (int(np.ceil(fftLen / 2.0 - l / 2.0)) - 1) if l % 2 == 1 else int(np.ceil(fftLen / 2.0 - l / 2.0)) - N = int(l) - - if isinstance(window, str): - sig = get_window(window, N, fftbins=True) - elif isinstance(window, tuple): - if window[0] == "gaussian": - assert window[1] >= 0 - sig = get_window(("gaussian", np.floor(-N / 2 / np.sqrt(-2 * np.log(10 ** (-window[1] / 20))))), N, fftbins=True) - else: raise Exception - - sig = sig * np.exp(np.r_[-l // 2: l // 2] * 1j * 2 * np.pi * freqs[k] / fs) / l - - if norm: - tempKernel[k, start: start + int(l)] = sig / np.linalg.norm(sig, norm) - else: - tempKernel[k, start: start + int(l)] = sig - - return tempKernel, fftLen, torch.tensor(lengths).float(), freqs - - @torch.no_grad() - def init_weights(self): - self.conv.weight.copy_(torch.cat((self.cqt_kernels.real, -self.cqt_kernels.imag), dim=0)) - self.conv.weight.requires_grad = self.trainable - - def forward(self, x, output_format=None, normalization_type="librosa"): - output_format = output_format or self.output_format - x = self.broadcast_dim(x) - cqt = self.conv(x).view(x.size(0), 2, self.n_bins, -1) - - if normalization_type == "librosa": cqt *= self.sqrt_lengths.to(cqt.device) - elif normalization_type == "convolutional": pass - elif normalization_type == "wrap": cqt *= 2 - else: raise ValueError - - if output_format == "Magnitude": return cqt.pow(2).sum(-3).add(1e-8 if self.trainable else 0).sqrt() - if output_format == "Complex": return cqt.permute(0, 2, 3, 1) - - cqt_real, cqt_imag = cqt.split(self.n_bins, dim=-2) - if output_format == "Phase": return torch.stack((cqt_imag.atan2(cqt_real).cos(), cqt_imag.atan2(cqt_real).sin()), -1) - - raise ValueError - - def broadcast_dim(self, x): - if x.dim() == 2: x = x[:, None, :] - elif x.dim() == 1: x = x[None, None, :] - elif x.dim() == 3: pass - else: raise ValueError - - return x - -class RegularCQT(BaseCQT): - def __init__(self, *args, pad_mode="reflect", **kwargs): - super().__init__(*args, **kwargs) - padding = self.kernel_width // 2 if self.center else 0 - self.conv = torch.nn.Conv1d( - 1, - 2 * self.n_bins, - kernel_size=self.kernel_width, - stride=self.hop_length, - padding=padding, - padding_mode=pad_mode, - bias=False - ) - self.init_weights() - -class StreamingCQT(BaseCQT): - def __init__( - self, - *args, - mirror = 0, - max_batch_size = 1, - **kwargs - ): - super(StreamingCQT, self).__init__(*args, **kwargs) - if self.center: - mirrored_samples = int(mirror * (self.kernel_width - self.hop_length) / 2) - padding = self.kernel_width - self.hop_length - mirrored_samples - else: - mirrored_samples = 0 - padding = 0 - - self.conv = CachedConv1d( - 1, - 2 * self.n_bins, - kernel_size=self.kernel_width, - stride=self.hop_length, - padding=padding, - mirror=mirrored_samples, - max_batch_size=max_batch_size, - bias=False - ) - - self.init_weights() - -class CQT: - regular_only_kwargs = ["pad_mode"] - streaming_only_kwargs = ["mirror", "max_batch_size"] - - def __new__(cls, *args, **kwargs): - streaming = kwargs.pop("streaming", False) - - if streaming: - for kwarg in cls.regular_only_kwargs: - kwargs.pop(kwarg, None) - - return StreamingCQT(*args, **kwargs) - - for kwarg in cls.streaming_only_kwargs: - kwargs.pop(kwarg, None) - - return RegularCQT(*args, **kwargs) - -class CachedConv1d(torch.nn.Conv1d): - def __init__( - self, - *args, - **kwargs - ): - kwargs["padding"] = 0 - super(CachedConv1d, self).__init__(*args, **kwargs) - padding = kwargs.get("padding", 0) - max_batch_size = kwargs.pop("max_batch_size", 1) - mirror = kwargs.pop("mirror", 0) - mirror_fn = kwargs.pop("mirror_fn", "zeros") - cumulative_delay = kwargs.pop("cumulative_delay", 0) - - if isinstance(padding, int): r_pad = padding - elif isinstance(padding, list) or isinstance(padding, tuple): - r_pad = padding[1] - padding = padding[0] + padding[1] - else: raise TypeError - - s = self.stride[0] - cd = cumulative_delay - - self.cumulative_delay = (r_pad + ((s - ((r_pad + cd) % s)) % s) + cd) // s - self.cache = CachedPadding1d(padding, max_batch_size=max_batch_size) - - if mirror == 0: - mirroring_fn = torch.nn.Identity - elif mirror_fn == "reflection": - mirroring_fn = torch.nn.ReflectionPad1d - elif mirror_fn == "zeros": - mirroring_fn = torch.nn.ZeroPad1d - elif mirror_fn == "refill": - mirroring_fn = RefillPad1d - else: - mirroring_fn = torch.nn.Identity - - self.mirror = mirroring_fn((0, mirror)) - - def forward(self, x): - return super(CachedConv1d, self).forward(self.mirror(self.cache(x))) - -class RefillPad1d(torch.nn.Module): - def __init__( - self, - padding - ): - super(RefillPad1d, self).__init__() - self.right_padding = padding[1] - - def forward(self, x): - return torch.cat((x, x[..., -self.right_padding:]), dim=-1) - -class CachedPadding1d(torch.nn.Module): - def __init__( - self, - padding, - max_batch_size = 1, - crop=False - ): - super().__init__() - self.padding = padding - self.max_batch_size = max_batch_size - self.crop = crop - self.init_cache() - - @torch.jit.unused - @torch.no_grad() - def init_cache(self): - self.register_buffer("pad", torch.zeros(self.max_batch_size, 1, self.padding), persistent=False) - - def forward(self, x): - bs = x.size(0) - if self.padding: - x = torch.cat((self.pad[:bs], x), -1) - self.pad[:bs].copy_(x[..., -self.padding:]) - - return x \ No newline at end of file diff --git a/main/library/predictors/RMVPE/RMVPE.py b/main/library/predictors/RMVPE/RMVPE.py deleted file mode 100644 index 0bb936791100093a8a7d859e59654f202fa2671b..0000000000000000000000000000000000000000 --- a/main/library/predictors/RMVPE/RMVPE.py +++ /dev/null @@ -1,121 +0,0 @@ -import os -import sys -import torch - -import numpy as np -import torch.nn.functional as F - -sys.path.append(os.getcwd()) - -from main.library.predictors.RMVPE.mel import MelSpectrogram - -N_MELS, N_CLASS = 128, 360 - -class RMVPE: - def __init__(self, model_path, is_half, device=None, providers=None, onnx=False, hpa=False, compile_model=False, compile_mode=None, enable_chunk = False, chunk_size = 8000, return_tensor = False, f0_min = 50, f0_max = 1100): - if onnx: - import onnxruntime as ort - - sess_options = ort.SessionOptions() - sess_options.log_severity_level = 3 - model = ort.InferenceSession(model_path, sess_options=sess_options, providers=providers) - else: - from main.library.predictors.RMVPE.e2e import E2E - - model = E2E(4, 1, (2, 2), 5, 4, 1, 16, hpa=hpa) - model.load_state_dict(torch.load(model_path, map_location="cpu", weights_only=True)) - model.to(device).eval() - - if is_half: model = model.half() - if compile_model: model = torch.compile(model, mode=compile_mode) - - self.model = model - self.f0_min = f0_min - self.f0_max = f0_max - self.device = device - self.chunk_size = chunk_size - self.dtype = torch.float16 if is_half else torch.float32 - - self.cents_mapping = np.pad(20 * np.arange(N_CLASS) + 1997.3794084376191, (4, 4)) - self.mel_extractor = MelSpectrogram(N_MELS, 16000, 1024, 160, 1024, 30, 8000).to(device) - if return_tensor: self.cents_mapping = torch.as_tensor(self.cents_mapping, dtype=self.dtype, device=device) - - self.mel2hidden = self._mel2hidden_chunk if enable_chunk else self._mel2hidden - self.offsets = torch.arange(-4, 5, device=device) if return_tensor else np.arange(-4, 5) - self.infer = self._infer_onnx if onnx else (self._infer_torch_fp16 if is_half else self._infer_torch_fp32) - self.to_local_average_cents = self._to_local_average_cents_tensor if return_tensor else self._to_local_average_cents_array - - def decode(self, hidden, thred=0.03): - f0 = 10 * (2 ** (self.to_local_average_cents(hidden, thred=thred) / 1200)) - f0[f0 == 10] = 0 - - return f0 - - def infer_from_audio(self, audio, thred=0.03): - audio = torch.from_numpy(audio).float().to(self.device) if not torch.is_tensor(audio) else audio - hidden = self.mel2hidden(self.mel_extractor(audio.unsqueeze(0), center=True)).squeeze(0) - - return self.decode(hidden, thred=thred) - - def infer_from_audio_with_pitch(self, audio, thred=0.03): - f0 = self.infer_from_audio(audio, thred) - f0[(f0 < self.f0_min) | (f0 > self.f0_max)] = 0 - - return f0 - - def _mel2hidden(self, mel): - with torch.no_grad(): - n_frames = mel.shape[-1] - return self.infer(F.pad(mel, (0, 32 * ((n_frames - 1) // 32 + 1) - n_frames), mode="reflect"))[:, :n_frames] - - def _mel2hidden_chunk(self, mel): - with torch.no_grad(): - n_frames = mel.shape[-1] - mel = F.pad(mel, (0, 32 * ((n_frames - 1) // 32 + 1) - n_frames), mode="reflect") - hidden = torch.cat([self.infer(mel[..., start:min(start + self.chunk_size, mel.shape[-1])]) for start in range(0, mel.shape[-1], self.chunk_size)], dim=1) - - return hidden[:, :n_frames] - - def _infer_torch_fp32(self, mel): - return self.model(mel) - - def _infer_torch_fp16(self, mel): - return self.model(mel.half()) - - def _infer_onnx(self, mel): - mel = mel.cpu().numpy().astype(np.float32) - - return torch.as_tensor( - self.model.run( - [self.model.get_outputs()[0].name], - {self.model.get_inputs()[0].name: mel} - )[0], - device=self.device, - dtype=self.dtype - ) - - def _to_local_average_cents_array(self, salience, thred=0.05): - salience = salience.cpu().numpy().astype(np.float32) - - center = np.argmax(salience, axis=1) - salience = np.pad(salience, ((0, 0), (4, 4))) - center += 4 - - idx = center[:, None] + self.offsets[None, :] - local_salience = salience[np.arange(salience.shape[0])[:, None], idx] - devided = np.sum(local_salience * self.cents_mapping[idx], axis=1) / np.sum(local_salience, axis=1) - devided[np.max(salience, axis=1) <= thred] = 0 - - return devided - - def _to_local_average_cents_tensor(self, salience, thred=0.05): - center = torch.argmax(salience, dim=1) - salience = F.pad(salience, (4, 4)) - center += 4 - - idx = center[:, None] + self.offsets[None, :] - local_salience = salience[torch.arange(salience.shape[0], device=salience.device)[:, None], idx] - devided = (local_salience * self.cents_mapping[idx]).sum(dim=1) / local_salience.sum(dim=1) - devided = torch.where(salience.max(dim=1).values <= thred, torch.zeros_like(devided), devided) - - return devided \ No newline at end of file diff --git a/main/library/predictors/RMVPE/deepunet.py b/main/library/predictors/RMVPE/deepunet.py deleted file mode 100644 index 25c0277acc00a327447dee9265db3759df476fa4..0000000000000000000000000000000000000000 --- a/main/library/predictors/RMVPE/deepunet.py +++ /dev/null @@ -1,352 +0,0 @@ -import os -import sys - -import torch - -import torch.nn as nn - -sys.path.append(os.getcwd()) - -from main.library.predictors.RMVPE.yolo import YOLO13Encoder, YOLO13FullPADDecoder, HyperACE - -class ConvBlockRes(nn.Module): - def __init__( - self, - in_channels, - out_channels, - momentum=0.01 - ): - super(ConvBlockRes, self).__init__() - self.conv = nn.Sequential( - nn.Conv2d( - in_channels=in_channels, - out_channels=out_channels, - kernel_size=(3, 3), - stride=(1, 1), - padding=(1, 1), - bias=False - ), - nn.BatchNorm2d( - out_channels, - momentum=momentum - ), - nn.ReLU(), - nn.Conv2d( - in_channels=out_channels, - out_channels=out_channels, - kernel_size=(3, 3), - stride=(1, 1), - padding=(1, 1), - bias=False - ), - nn.BatchNorm2d( - out_channels, - momentum=momentum - ), - nn.ReLU() - ) - - self.is_shortcut = False - if in_channels != out_channels: - self.shortcut = nn.Conv2d(in_channels, out_channels, (1, 1)) - self.is_shortcut = True - - self.forward = self.forward_shortcut if self.is_shortcut else self.forward_non_shortcut - - def forward_shortcut(self, x): - return self.conv(x) + self.shortcut(x) - - def forward_non_shortcut(self, x): - return self.conv(x) + x - -class ResEncoderBlock(nn.Module): - def __init__( - self, - in_channels, - out_channels, - kernel_size, - n_blocks=1, - momentum=0.01 - ): - super(ResEncoderBlock, self).__init__() - self.n_blocks = n_blocks - self.conv = nn.ModuleList() - self.conv.append( - ConvBlockRes( - in_channels, - out_channels, - momentum - ) - ) - - for _ in range(n_blocks - 1): - self.conv.append( - ConvBlockRes( - out_channels, - out_channels, - momentum - ) - ) - - self.kernel_size = kernel_size - if self.kernel_size is not None: self.pool = nn.AvgPool2d(kernel_size=kernel_size) - - def forward(self, x): - for i in range(self.n_blocks): - x = self.conv[i](x) - - if self.kernel_size is not None: return x, self.pool(x) - else: return x - -class Encoder(nn.Module): - def __init__( - self, - in_channels, - in_size, - n_encoders, - kernel_size, - n_blocks, - out_channels=16, - momentum=0.01 - ): - super(Encoder, self).__init__() - self.n_encoders = n_encoders - self.bn = nn.BatchNorm2d(in_channels, momentum=momentum) - self.layers = nn.ModuleList() - - for _ in range(self.n_encoders): - self.layers.append( - ResEncoderBlock( - in_channels, - out_channels, - kernel_size, - n_blocks, - momentum=momentum - ) - ) - - in_channels = out_channels - out_channels *= 2 - in_size //= 2 - - self.out_size = in_size - self.out_channel = out_channels - - def forward(self, x): - concat_tensors = [] - x = self.bn(x) - - for layer in self.layers: - t, x = layer(x) - concat_tensors.append(t) - - return x, concat_tensors - -class Intermediate(nn.Module): - def __init__( - self, - in_channels, - out_channels, - n_inters, - n_blocks, - momentum=0.01 - ): - super(Intermediate, self).__init__() - self.layers = nn.ModuleList() - self.layers.append( - ResEncoderBlock( - in_channels, - out_channels, - None, - n_blocks, - momentum - ) - ) - - for _ in range(n_inters - 1): - self.layers.append( - ResEncoderBlock( - out_channels, - out_channels, - None, - n_blocks, - momentum - ) - ) - - def forward(self, x): - for layer in self.layers: - x = layer(x) - - return x - -class ResDecoderBlock(nn.Module): - def __init__( - self, - in_channels, - out_channels, - stride, - n_blocks=1, - momentum=0.01 - ): - super(ResDecoderBlock, self).__init__() - out_padding = (0, 1) if stride == (1, 2) else (1, 1) - self.conv1 = nn.Sequential( - nn.ConvTranspose2d( - in_channels=in_channels, - out_channels=out_channels, - kernel_size=(3, 3), - stride=stride, - padding=(1, 1), - output_padding=out_padding, - bias=False - ), - nn.BatchNorm2d( - out_channels, - momentum=momentum - ), - nn.ReLU() - ) - - self.conv2 = nn.ModuleList() - self.conv2.append( - ConvBlockRes( - out_channels * 2, - out_channels, - momentum - ) - ) - - for _ in range(n_blocks - 1): - self.conv2.append( - ConvBlockRes( - out_channels, - out_channels, - momentum - ) - ) - - def forward(self, x, concat_tensor): - x = torch.cat((self.conv1(x), concat_tensor), dim=1) - for conv2 in self.conv2: - x = conv2(x) - - return x - -class Decoder(nn.Module): - def __init__( - self, - in_channels, - n_decoders, - stride, - n_blocks, - momentum=0.01 - ): - super(Decoder, self).__init__() - self.layers = nn.ModuleList() - - for _ in range(n_decoders): - out_channels = in_channels // 2 - self.layers.append( - ResDecoderBlock( - in_channels, - out_channels, - stride, - n_blocks, - momentum - ) - ) - in_channels = out_channels - - def forward(self, x, concat_tensors): - for i, layer in enumerate(self.layers): - x = layer(x, concat_tensors[-1 - i]) - - return x - -class DeepUnet(nn.Module): - def __init__( - self, - kernel_size, - n_blocks, - en_de_layers=5, - inter_layers=4, - in_channels=1, - en_out_channels=16 - ): - super(DeepUnet, self).__init__() - self.encoder = Encoder( - in_channels, - 128, - en_de_layers, - kernel_size, - n_blocks, - en_out_channels - ) - self.intermediate = Intermediate( - self.encoder.out_channel // 2, - self.encoder.out_channel, - inter_layers, - n_blocks - ) - self.decoder = Decoder( - self.encoder.out_channel, - en_de_layers, - kernel_size, - n_blocks - ) - - def forward(self, x): - x, concat_tensors = self.encoder(x) - - return self.decoder( - self.intermediate(x), - concat_tensors - ) - -class HPADeepUnet(nn.Module): - def __init__( - self, - in_channels=1, - en_out_channels=16, - base_channels=64, - hyperace_k=2, - hyperace_l=1, - num_hyperedges=16, - num_heads=8 - ): - super().__init__() - self.encoder = YOLO13Encoder( - in_channels, - base_channels - ) - - enc_ch = self.encoder.out_channels - - self.hyperace = HyperACE( - in_channels=enc_ch, - out_channels=enc_ch[-1], - num_hyperedges=num_hyperedges, - num_heads=num_heads, - k=hyperace_k, - l=hyperace_l - ) - - self.decoder = YOLO13FullPADDecoder( - encoder_channels=enc_ch, - hyperace_out_c=enc_ch[-1], - out_channels_final=en_out_channels - ) - - def forward(self, x): - features = self.encoder(x) - - return nn.functional.interpolate( - self.decoder( - features, - self.hyperace(features) - ), - size=x.shape[2:], - mode='bilinear', - align_corners=False - ) \ No newline at end of file diff --git a/main/library/predictors/RMVPE/e2e.py b/main/library/predictors/RMVPE/e2e.py deleted file mode 100644 index 2b72b2c1d567ca0fa769aa7288d4d52562e1d55e..0000000000000000000000000000000000000000 --- a/main/library/predictors/RMVPE/e2e.py +++ /dev/null @@ -1,93 +0,0 @@ -import os -import sys -import torch - -import torch.nn as nn - -sys.path.append(os.getcwd()) - -from main.library.predictors.RMVPE.deepunet import DeepUnet, HPADeepUnet - -N_MELS, N_CLASS = 128, 360 - -class BiGRU(nn.Module): - def __init__( - self, - input_features, - hidden_features, - num_layers - ): - super(BiGRU, self).__init__() - self.gru = nn.GRU( - input_features, - hidden_features, - num_layers=num_layers, - batch_first=True, - bidirectional=True - ) - - def forward(self, x): - try: - return self.gru(x)[0] - except: - torch.backends.cudnn.enabled = False - return self.gru(x)[0] - -class E2E(nn.Module): - def __init__( - self, - n_blocks, - n_gru, - kernel_size, - en_de_layers=5, - inter_layers=4, - in_channels=1, - en_out_channels=16, - hpa=False - ): - super(E2E, self).__init__() - self.unet = ( - HPADeepUnet( - in_channels=in_channels, - en_out_channels=en_out_channels, - base_channels=64, - hyperace_k=2, - hyperace_l=1, - num_hyperedges=16, - num_heads=4 - ) - ) if hpa else ( - DeepUnet( - kernel_size, - n_blocks, - en_de_layers, - inter_layers, - in_channels, - en_out_channels - ) - ) - - self.cnn = nn.Conv2d(en_out_channels, 3, (3, 3), padding=(1, 1)) - self.fc = ( - nn.Sequential( - BiGRU(3 * 128, 256, n_gru), - nn.Linear(512, N_CLASS), - nn.Dropout(0.25), - nn.Sigmoid() - ) - ) if n_gru else ( - nn.Sequential( - nn.Linear(3 * N_MELS, N_CLASS), - nn.Dropout(0.25), - nn.Sigmoid() - ) - ) - - def forward(self, mel): - return self.fc( - self.cnn( - self.unet( - mel.transpose(-1, -2).unsqueeze(1) - ) - ).transpose(1, 2).flatten(-2) - ) \ No newline at end of file diff --git a/main/library/predictors/RMVPE/mel.py b/main/library/predictors/RMVPE/mel.py deleted file mode 100644 index 1c46662f12cd9d1e162231a87ae31b8e595e494e..0000000000000000000000000000000000000000 --- a/main/library/predictors/RMVPE/mel.py +++ /dev/null @@ -1,62 +0,0 @@ -import os -import sys -import torch - -import torch.nn as nn - -sys.path.append(os.getcwd()) - -from main.app.variables import config -from main.library.audio.features import mel - -class MelSpectrogram(nn.Module): - def __init__( - self, - n_mel_channels, - sample_rate, - win_length, - hop_length, - n_fft=None, - mel_fmin=0, - mel_fmax=None, - clamp=1e-5 - ): - super().__init__() - self.clamp = clamp - self.n_fft = n_fft - self.hop_length = hop_length - self.win_length = win_length - self.sample_rate = sample_rate - self.n_mel_channels = n_mel_channels - self.hann_window = torch.hann_window(win_length).to(config.device) - self.stftt = self._stft_other_backends if config.device.startswith(("ocl", "privateuseone")) else self._stft_torch - self.register_buffer("mel_basis", mel(sr=sample_rate, n_fft=n_fft, n_mels=n_mel_channels, fmin=mel_fmin, fmax=mel_fmax, htk=True, device=config.device)) - - def forward(self, audio, center=True): - mel_output = self.mel_basis @ self.stftt(audio, center) - return mel_output.clamp(min=self.clamp).log() - - def _stft_other_backends(self, audio, center=True): - if not hasattr(self, "stft"): - from main.library.backends.utils import STFT - - self.stft = STFT( - filter_length=self.n_fft, - hop_length=self.hop_length, - win_length=self.win_length, - center=center - ).to(audio.device) - - return self.stft.transform(audio, 1e-9) - - def _stft_torch(self, audio, center=True): - fft = torch.stft( - audio, - n_fft=self.n_fft, - hop_length=self.hop_length, - win_length=self.win_length, - window=self.hann_window, - center=center, - return_complex=True - ) - return fft.abs() \ No newline at end of file diff --git a/main/library/predictors/RMVPE/yolo.py b/main/library/predictors/RMVPE/yolo.py deleted file mode 100644 index cf6a4084dbffa21bdc8a15e74971d22c8294753d..0000000000000000000000000000000000000000 --- a/main/library/predictors/RMVPE/yolo.py +++ /dev/null @@ -1,514 +0,0 @@ -import torch -import torch.nn as nn -import torch.nn.functional as F - -def autopad(k, p=None): - if p is None: p = k // 2 if isinstance(k, int) else [x // 2 for x in k] - return p - -class Conv(nn.Module): - def __init__( - self, - c1, - c2, - k=1, - s=1, - p=None, - g=1, - act=True - ): - super().__init__() - self.conv = nn.Conv2d(c1, c2, k, s, autopad(k, p), groups=g, bias=False) - self.bn = nn.BatchNorm2d(c2) - self.act = nn.SiLU() if act else nn.Identity() - - def forward(self, x): - return self.act( - self.bn( - self.conv(x) - ) - ) - -class DSConv(nn.Module): - def __init__( - self, - c1, - c2, - k=3, - s=1, - p=None, - act=True - ): - super().__init__() - self.dwconv = nn.Conv2d(c1, c1, k, s, autopad(k, p), groups=c1, bias=False) - self.pwconv = nn.Conv2d(c1, c2, 1, 1, 0, bias=False) - self.bn = nn.BatchNorm2d(c2) - self.act = nn.SiLU() if act else nn.Identity() - - def forward(self, x): - return self.act( - self.bn( - self.pwconv( - self.dwconv(x) - ) - ) - ) - -class DS_Bottleneck(nn.Module): - def __init__( - self, - c1, - c2, - k=3, - shortcut=True - ): - super().__init__() - self.dsconv1 = DSConv(c1, c1, k=3, s=1) - self.dsconv2 = DSConv(c1, c2, k=k, s=1) - self.shortcut = shortcut and c1 == c2 - - def forward(self, x): - return x + self.dsconv2(self.dsconv1(x)) if self.shortcut else self.dsconv2(self.dsconv1(x)) - -class DS_C3k(nn.Module): - def __init__( - self, - c1, - c2, - n=1, - k=3, - e=0.5 - ): - super().__init__() - self.cv1 = Conv(c1, int(c2 * e), 1, 1) - self.cv2 = Conv(c1, int(c2 * e), 1, 1) - self.cv3 = Conv(2 * int(c2 * e), c2, 1, 1) - self.m = nn.Sequential( - *[ - DS_Bottleneck( - int(c2 * e), - int(c2 * e), - k=k, - shortcut=True - ) - for _ in range(n) - ] - ) - - def forward(self, x): - return self.cv3( - torch.cat( - (self.m(self.cv1(x)), self.cv2(x)), - dim=1 - ) - ) - -class DS_C3k2(nn.Module): - def __init__( - self, - c1, - c2, - n=1, - k=3, - e=0.5 - ): - super().__init__() - self.cv1 = Conv(c1, int(c2 * e), 1, 1) - self.m = DS_C3k(int(c2 * e), int(c2 * e), n=n, k=k, e=1.0) - self.cv2 = Conv(int(c2 * e), c2, 1, 1) - - def forward(self, x): - return self.cv2( - self.m( - self.cv1(x) - ) - ) - -class AdaptiveHyperedgeGeneration(nn.Module): - def __init__( - self, - in_channels, - num_hyperedges, - num_heads - ): - super().__init__() - self.num_hyperedges = num_hyperedges - self.num_heads = num_heads - self.head_dim = max(1, in_channels // num_heads) - self.global_proto = nn.Parameter(torch.randn(num_hyperedges, in_channels)) - self.context_mapper = nn.Linear(2 * in_channels, num_hyperedges * in_channels, bias=False) - self.query_proj = nn.Linear(in_channels, in_channels, bias=False) - self.scale = self.head_dim ** -0.5 - - def forward(self, x): - B, N, C = x.shape - P = ( - self.global_proto.unsqueeze(0) + - self.context_mapper( - torch.cat( - ( - F.adaptive_avg_pool1d(x.permute(0, 2, 1), 1).squeeze(-1), - F.adaptive_max_pool1d(x.permute(0, 2, 1), 1).squeeze(-1) - ), - dim=1 - ) - ).view(B, self.num_hyperedges, C)) - - return F.softmax(( - (self.query_proj(x).view(B, N, self.num_heads, self.head_dim).permute(0, 2, 1, 3) @ P.view(B, self.num_hyperedges, self.num_heads, self.head_dim).permute(0, 2, 3, 1)) * self.scale - ).mean(dim=1).permute(0, 2, 1), dim=-1) - -class HypergraphConvolution(nn.Module): - def __init__( - self, - in_channels, - out_channels - ): - super().__init__() - self.W_e = nn.Linear(in_channels, in_channels, bias=False) - self.W_v = nn.Linear(in_channels, out_channels, bias=False) - self.act = nn.SiLU() - - def forward(self, x, A): - return x + self.act(self.W_v(A.transpose(1, 2).bmm(self.act(self.W_e(A.bmm(x)))))) - -class AdaptiveHypergraphComputation(nn.Module): - def __init__( - self, - in_channels, - out_channels, - num_hyperedges, - num_heads - ): - super().__init__() - self.adaptive_hyperedge_gen = AdaptiveHyperedgeGeneration(in_channels, num_hyperedges, num_heads) - self.hypergraph_conv = HypergraphConvolution(in_channels, out_channels) - - def forward(self, x): - B, _, H, W = x.shape - x_flat = x.flatten(2).permute(0, 2, 1) - - return self.hypergraph_conv(x_flat, self.adaptive_hyperedge_gen(x_flat)).permute(0, 2, 1).view(B, -1, H, W) - -class C3AH(nn.Module): - def __init__( - self, - c1, - c2, - num_hyperedges, - num_heads, - e=0.5 - ): - super().__init__() - self.cv1 = Conv(c1, int(c1 * e), 1, 1) - self.cv2 = Conv(c1, int(c1 * e), 1, 1) - self.ahc = AdaptiveHypergraphComputation(int(c1 * e), int(c1 * e), num_hyperedges, num_heads) - self.cv3 = Conv(2 * int(c1 * e), c2, 1, 1) - - def forward(self, x): - return self.cv3( - torch.cat( - (self.ahc(self.cv2(x)), self.cv1(x)), - dim=1 - ) - ) - -class HyperACE(nn.Module): - def __init__( - self, - in_channels, - out_channels, - num_hyperedges=16, - num_heads=8, - k=2, - l=1, - c_h=0.5, - c_l=0.25 - ): - super().__init__() - c2, c3, c4, c5 = in_channels - c_mid = c4 - self.fuse_conv = Conv(c2 + c3 + c4 + c5, c_mid, 1, 1) - self.c_h = int(c_mid * c_h) - self.c_l = int(c_mid * c_l) - self.c_s = c_mid - self.c_h - self.c_l - self.high_order_branch = nn.ModuleList([ - C3AH( - self.c_h, - self.c_h, - num_hyperedges=num_hyperedges, - num_heads=num_heads, e=1.0 - ) - for _ in range(k) - ]) - self.high_order_fuse = Conv(self.c_h * k, self.c_h, 1, 1) - self.low_order_branch = nn.Sequential( - *[ - DS_C3k( - self.c_l, - self.c_l, - n=1, - k=3, - e=1.0 - ) - for _ in range(l) - ] - ) - self.final_fuse = Conv(self.c_h + self.c_l + self.c_s, out_channels, 1, 1) - - def forward(self, x): - B2, B3, B4, B5 = x - _, _, H4, W4 = B4.shape - - x_h, x_l, x_s = self.fuse_conv( - torch.cat( - ( - F.interpolate( - B2, - size=(H4, W4), - mode='bilinear', - align_corners=False - ), - F.interpolate( - B3, - size=(H4, W4), - mode='bilinear', - align_corners=False - ), - B4, - F.interpolate( - B5, - size=(H4, W4), - mode='bilinear', - align_corners=False - ) - ), - dim=1 - ) - ).split([self.c_h, self.c_l, self.c_s], dim=1) - - return self.final_fuse( - torch.cat( - ( - self.high_order_fuse(torch.cat([m(x_h) for m in self.high_order_branch], dim=1)), - self.low_order_branch(x_l), - x_s - ), - dim=1 - ) - ) - -class GatedFusion(nn.Module): - def __init__( - self, - in_channels - ): - super().__init__() - self.gamma = nn.Parameter(torch.zeros(1, in_channels, 1, 1)) - - def forward(self, f_in, h): - return f_in + self.gamma * h - -class YOLO13Encoder(nn.Module): - def __init__( - self, - in_channels, - base_channels=32 - ): - super().__init__() - self.stem = DSConv( - in_channels, - base_channels, - k=3, - s=1 - ) - self.p2 = nn.Sequential( - DSConv( - base_channels, - base_channels*2, k=3, s=(2, 2)), - DS_C3k2( - base_channels*2, - base_channels*2, - n=1 - ) - ) - self.p3 = nn.Sequential( - DSConv( - base_channels*2, - base_channels*4, - k=3, - s=(2, 2) - ), - DS_C3k2( - base_channels*4, - base_channels*4, - n=2 - ) - ) - self.p4 = nn.Sequential( - DSConv( - base_channels*4, - base_channels*8, - k=3, - s=(2, 2) - ), - DS_C3k2( - base_channels*8, - base_channels*8, - n=2 - ) - ) - self.p5 = nn.Sequential( - DSConv( - base_channels*8, - base_channels*16, - k=3, - s=(2, 2) - ), - DS_C3k2( - base_channels*16, - base_channels*16, - n=1 - ) - ) - - self.out_channels = [base_channels*2, base_channels*4, base_channels*8, base_channels*16] - - def forward(self, x): - p2 = self.p2(self.stem(x)) - p3 = self.p3(p2) - p4 = self.p4(p3) - p5 = self.p5(p4) - - return [p2, p3, p4, p5] - -class YOLO13FullPADDecoder(nn.Module): - def __init__(self, encoder_channels, hyperace_out_c, out_channels_final): - super().__init__() - c_p2, c_p3, c_p4, c_p5 = encoder_channels - c_d5, c_d4, c_d3, c_d2 = c_p5, c_p4, c_p3, c_p2 - - self.h_to_d5 = Conv( - hyperace_out_c, - c_d5, - 1, - 1 - ) - self.h_to_d4 = Conv( - hyperace_out_c, - c_d4, - 1, - 1 - ) - self.h_to_d3 = Conv( - hyperace_out_c, - c_d3, - 1, - 1 - ) - self.h_to_d2 = Conv( - hyperace_out_c, - c_d2, - 1, - 1 - ) - - self.fusion_d5 = GatedFusion(c_d5) - self.fusion_d4 = GatedFusion(c_d4) - self.fusion_d3 = GatedFusion(c_d3) - self.fusion_d2 = GatedFusion(c_d2) - - self.skip_p5 = Conv( - c_p5, - c_d5, - 1, - 1 - ) - self.skip_p4 = Conv( - c_p4, - c_d4, - 1, - 1 - ) - self.skip_p3 = Conv( - c_p3, - c_d3, - 1, - 1 - ) - self.skip_p2 = Conv( - c_p2, - c_d2, - 1, - 1 - ) - - self.up_d5 = DS_C3k2( - c_d5, - c_d4, - n=1 - ) - self.up_d4 = DS_C3k2( - c_d4, - c_d3, - n=1 - ) - self.up_d3 = DS_C3k2( - c_d3, - c_d2, - n=1 - ) - - self.final_d2 = DS_C3k2( - c_d2, - c_d2, - n=1 - ) - self.final_conv = Conv( - c_d2, - out_channels_final, - 1, - 1 - ) - - def forward(self, enc_feats, h_ace): - p2, p3, p4, p5 = enc_feats - d5 = self.skip_p5(p5) - - d4 = self.up_d5( - F.interpolate( - self.fusion_d5(d5, self.h_to_d5(F.interpolate(h_ace, size=d5.shape[2:], mode='bilinear', align_corners=False))), - size=p4.shape[2:], - mode='bilinear', - align_corners=False - ) - ) + self.skip_p4(p4) - - d3 = self.up_d4( - F.interpolate( - self.fusion_d4(d4, self.h_to_d4(F.interpolate(h_ace, size=d4.shape[2:], mode='bilinear', align_corners=False))), - size=p3.shape[2:], - mode='bilinear', - align_corners=False - ) - ) + self.skip_p3(p3) - - d2 = self.up_d3( - F.interpolate( - self.fusion_d3(d3, self.h_to_d3(F.interpolate(h_ace, size=d3.shape[2:], mode='bilinear', align_corners=False))), - size=p2.shape[2:], - mode='bilinear', - align_corners=False - ) - ) + self.skip_p2(p2) - - return self.final_conv( - self.final_d2( - self.fusion_d2( - d2, - self.h_to_d2( - F.interpolate(h_ace, size=d2.shape[2:], mode='bilinear', align_corners=False) - ) - ) - ) - ) \ No newline at end of file diff --git a/main/library/predictors/SWIFT/SWIFT.py b/main/library/predictors/SWIFT/SWIFT.py deleted file mode 100644 index 362fd9c026a4742ca6fe39fbd292ff98b4235f32..0000000000000000000000000000000000000000 --- a/main/library/predictors/SWIFT/SWIFT.py +++ /dev/null @@ -1,41 +0,0 @@ -import onnxruntime - -import numpy as np - -SAMPLE_RATE, HOP_LENGTH, FRAME_LENGTH = 16000, 256, 1024 - -class SWIFT: - def __init__( - self, - model_path, - fmin = 50, - fmax = 1100, - confidence_threshold = 0.9, - providers = ["CPUExecutionProvider"] - ): - self.fmin = fmin - self.fmax = fmax - self.confidence_threshold = confidence_threshold - session_options = onnxruntime.SessionOptions() - session_options.inter_op_num_threads = 1 - session_options.intra_op_num_threads = 1 - self.pitch_session = onnxruntime.InferenceSession(model_path, session_options, providers=providers) - self.pitch_input_name = self.pitch_session.get_inputs()[0].name - - def _extract_pitch_and_confidence(self, audio_16k): - if len(audio_16k) < 256: audio_16k = np.pad(audio_16k, (0, max(0, 256 - len(audio_16k))), mode="constant") - outputs = self.pitch_session.run(None, {self.pitch_input_name: audio_16k[None, :].astype(np.float32)}) - return outputs[0][0], outputs[1][0] - - def _compute_voicing(self, pitch_hz, confidence): - return (confidence > self.confidence_threshold) & (pitch_hz >= self.fmin) & (pitch_hz <= self.fmax) - - def _calculate_timestamps(self, n_frames): - frame_centers = np.arange(n_frames) * HOP_LENGTH + ((FRAME_LENGTH - 1) / 2 - ((FRAME_LENGTH - HOP_LENGTH) // 2)) - return frame_centers / SAMPLE_RATE - - def detect_from_array(self, audio_array): - if audio_array.ndim > 1: audio_array = np.mean(audio_array, axis=-1) - pitch_hz, confidence = self._extract_pitch_and_confidence(audio_array) - - return pitch_hz, self._compute_voicing(pitch_hz, confidence), self._calculate_timestamps(len(pitch_hz)) \ No newline at end of file diff --git a/main/library/predictors/WORLD/SWIPE.py b/main/library/predictors/WORLD/SWIPE.py deleted file mode 100644 index 93adea7f79c6c6da42fcdf516e4bb112780562c9..0000000000000000000000000000000000000000 --- a/main/library/predictors/WORLD/SWIPE.py +++ /dev/null @@ -1,226 +0,0 @@ -import math - -import numba as nb -import numpy as np - -from matplotlib import mlab -from scipy import interpolate -from decimal import Decimal, ROUND_HALF_UP - -def swipe( - x, - fs, - f0_floor=50, - f0_ceil=1100, - frame_period=10, - sTHR=0.3 -): - plim = np.array([f0_floor, f0_ceil]) - t = np.arange(0, int(1000 * len(x) / fs / (frame_period) + 1)) * (frame_period / 1000) - - log2pc = np.arange(np.log2(plim[0]) * 96, np.log2(plim[-1]) * 96) - log2pc *= (1 / 96) - - pc = 2 ** log2pc - S = np.zeros((len(pc), len(t))) - - logWs = [round_matlab(elm) for elm in np.log2(4 * 2 * fs / plim)] - ws = 2 ** np.arange(logWs[0], logWs[1] - 1, -1) - p0 = 4 * 2 * fs / ws - - d = 1 + log2pc - np.log2(4 * 2 * fs / ws[0]) - fERBs = erbs2hz(np.arange(hz2erbs(pc[0] / 4), hz2erbs(fs / 2), 0.1)) - - for i in range(len(ws)): - dn = round_matlab(4 * fs / p0[i]) - - X, f, ti = mlab.specgram( - x=np.r_[np.zeros(int(ws[i] / 2)), np.r_[x, np.zeros(int(dn + ws[i] / 2))]], - NFFT=ws[i], - Fs=fs, - window=np.hanning(ws[i] + 2)[1:-1], noverlap=max(0, np.round(ws[i] - dn)), - mode='complex' - ) - - ti = np.r_[0, ti[:-1]] - M = np.maximum(0, interpolate.interp1d(f, np.abs(X.T), kind='cubic')(fERBs)).T - - if i == len(ws) - 1: - j = np.where(d - (i + 1) > -1)[0] - k = np.where(d[j] - (i + 1) < 0)[0] - elif i == 0: - j = np.where(d - (i + 1) < 1)[0] - k = np.where(d[j] - (i + 1) > 0)[0] - else: - j = np.where(np.abs(d - (i + 1)) < 1)[0] - k = np.arange(len(j)) - - Si = pitchStrengthAllCandidates(fERBs, np.sqrt(M), pc[j]) - Si = interpolate.interp1d( - ti, - Si, - bounds_error=False, - fill_value='nan' - )(t) if Si.shape[1] > 1 else np.full( - (len(Si), len(t)), - np.nan - ) - - mu = np.ones(j.shape) - mu[k] = 1 - np.abs(d[j[k]] - i - 1) - S[j, :] = S[j, :] + np.tile(mu.reshape(-1, 1), (1, Si.shape[1])) * Si - - - p = np.full((S.shape[1], 1), np.nan) - s = np.full((S.shape[1], 1), np.nan) - - for j in range(S.shape[1]): - s[j] = np.max(S[:, j]) - i = np.argmax(S[:, j]) - - if s[j] < sTHR: continue - - if i == 0: p[j] = pc[0] - elif i == len(pc) - 1: p[j] = pc[0] - else: - I = np.arange(i-1, i+2) - tc = 1 / pc[I] - - ntc = (tc / tc[1] - 1) * 2 * np.pi - idx = np.isfinite(S[I, j]) - - c = np.zeros(len(ntc)) - c += np.nan - - I_ = I[idx] - - if len(I_) < 2: c[idx] = (S[I, j])[0] / ntc[0] - else: c[idx] = np.polyfit(ntc[idx], (S[I_, j]), 2) - - pval = np.polyval( - c, - ((1 / (2 ** np.arange(np.log2(pc[I[0]]), np.log2(pc[I[2]]) + 1 / 12 / 64, 1 / 12 / 64))) / tc[1] - 1) * 2 * np.pi - ) - - s[j] = np.max(pval) - p[j] = 2 ** (np.log2(pc[I[0]]) + (np.argmax(pval)) / 12 / 64) - - p = p.flatten() - p[np.isnan(p)] = 0 - - return np.array(p, dtype=np.float32), np.array(t, dtype=np.float32) - -def round_matlab(n): - return int(Decimal(n).quantize(0, ROUND_HALF_UP)) - -def pitchStrengthAllCandidates(f, L, pc): - den = np.sqrt(np.sum(L * L, axis=0)) - den = np.where(den == 0, 2.220446049250313e-16, den) - - L = L / den - S = np.zeros((len(pc), L.shape[1])) - - for j in range(len(pc)): - S[j,:] = pitchStrengthOneCandidate(f, L, pc[j]) - - return S - -def pitchStrengthOneCandidate(f, L, pc): - k = np.zeros(len(f)) - q = f / pc - - for i in ([1] + sieve(int(np.fix(f[-1] / pc - 0.75)))): - a = np.abs(q - i) - p = a < 0.25 - k[p] = np.cos(2 * np.pi * q[p]) - - v = np.logical_and((0.25 < a), (a < 0.75)) - k[v] = k[v] + np.cos(2 * np.pi * q[v]) / 2 - - k *= np.sqrt(1 / f) - k /= np.linalg.norm(k[k>0]) - - return k @ L - -def hz2erbs(hz): - return 21.4 * np.log10(1 + hz / 229) - -def erbs2hz(erbs): - return (10 ** (erbs / 21.4) - 1) * 229 - -def sieve(n): - primes = list(range(2, n + 1)) - num = 2 - - while num < math.sqrt(n): - i = num - - while i <= n: - i += num - - if i in primes: primes.remove(i) - - for j in primes: - if j > num: - num = j - break - - return primes - -def stonemask(x, fs, temporal_positions, f0): - refined_f0 = np.copy(f0) - - for i in range(len(temporal_positions)): - if f0[i] != 0: - refined_f0[i] = get_refined_f0(x, fs, temporal_positions[i], f0[i]) - if abs(refined_f0[i] - f0[i]) / f0[i] > 0.2: refined_f0[i] = f0[i] - - return np.array(refined_f0, dtype=np.float32) - -def get_refined_f0(x, fs, current_time, current_f0): - f0_initial = current_f0 - half_window_length = np.ceil(3 * fs / f0_initial / 2) - window_length_in_time = (2 * half_window_length + 1) / fs - - base_time = np.arange(-half_window_length, half_window_length + 1) / fs - fft_size = 2 ** math.ceil(math.log((half_window_length * 2 + 1), 2) + 1) - - base_time = np.array([float("{0:.4f}".format(elm)) for elm in base_time]) - index_raw = round_matlab_2((current_time + base_time) * fs) - - window_time = ((index_raw - 1) / fs) - current_time - main_window = 0.42 + 0.5 * np.cos(2 * math.pi * window_time / window_length_in_time) + 0.08 * np.cos(4 * math.pi * window_time / window_length_in_time) - - index = np.array(np.maximum(1, np.minimum(len(x), index_raw)), dtype=int) - spectrum = np.fft.fft(x[index - 1] * main_window, fft_size) - - diff_spectrum = np.fft.fft(x[index - 1] * (-(np.diff(np.r_[0, main_window]) + np.diff(np.r_[main_window, 0])) / 2), fft_size) - power_spectrum = np.abs(spectrum) ** 2 - - from sys import float_info - - power_spectrum[power_spectrum == 0] = float_info.epsilon - instantaneous_frequency = (np.arange(fft_size) / fft_size * fs) + (np.real(spectrum) * np.imag(diff_spectrum) - np.imag(spectrum) * np.real(diff_spectrum)) / power_spectrum * fs / 2 / math.pi - - trim_index = np.array([1, 2]) - index_list_trim = np.array(round_matlab_2(f0_initial * fft_size / fs * trim_index) + 1, int) - - amp_list = np.sqrt(power_spectrum[index_list_trim - 1]) - f0_initial = np.sum(amp_list * instantaneous_frequency[index_list_trim - 1]) / np.sum(amp_list * trim_index) - - if f0_initial < 0: return 0 - - trim_index = np.array([1, 2, 3, 4, 5, 6]) - index_list_trim = np.array(round_matlab_2(f0_initial * fft_size / fs * trim_index) + 1, int) - amp_list = np.sqrt(power_spectrum[index_list_trim - 1]) - - return np.sum(amp_list * instantaneous_frequency[index_list_trim - 1]) / np.sum(amp_list * trim_index) - -@nb.jit((nb.float64[:],), nopython=True, cache=True) -def round_matlab_2(x): - y = x.copy() - - y[x > 0] += 0.5 - y[x <= 0] -= 0.5 - - return y \ No newline at end of file diff --git a/main/library/predictors/WORLD/WORLD.py b/main/library/predictors/WORLD/WORLD.py deleted file mode 100644 index 4659e3ca13f6dff89a1822972cc6e087846c6c5e..0000000000000000000000000000000000000000 --- a/main/library/predictors/WORLD/WORLD.py +++ /dev/null @@ -1,167 +0,0 @@ -import os -import pickle -import ctypes -import platform - -import numpy as np - -class DioOption(ctypes.Structure): - _fields_ = [ - ("F0Floor", ctypes.c_double), - ("F0Ceil", ctypes.c_double), - ("ChannelsInOctave", ctypes.c_double), - ("FramePeriod", ctypes.c_double), - ("Speed", ctypes.c_int), - ("AllowedRange", ctypes.c_double) - ] - -class HarvestOption(ctypes.Structure): - _fields_ = [ - ("F0Floor", ctypes.c_double), - ("F0Ceil", ctypes.c_double), - ("FramePeriod", ctypes.c_double) - ] - -class PYWORLD: - def __init__(self, world_path, model_path, harvest = True): - self.world_path = world_path - os.makedirs(self.world_path, exist_ok=True) - - if platform.system() == "Windows": - model_type, suffix = ( - ("world_64" if platform.architecture()[0] == "64bit" else "world_86"), ".dll" - ) - elif platform.system() == "Linux": - model_type, suffix = ( - "world_linux", ".so" - ) - elif platform.system() == "Darwin": - model_type, suffix = ( - "world_mac", ".dylib" - ) - else: - raise ValueError - - self.world_file_path = os.path.join(self.world_path, f"{model_type}{suffix}") - - if not os.path.exists(self.world_file_path): - with open(model_path, "rb") as f: - model = pickle.load(f) - - with open(self.world_file_path, "wb") as w: - w.write(model[model_type]) - - self.world_dll = ctypes.CDLL(self.world_file_path) - self.infer = self.harvest if harvest else self.dio - - def harvest( - self, - x, - fs, - f0_floor=50, - f0_ceil=1100, - frame_period=10 - ): - self.world_dll.Harvest.argtypes = [ - ctypes.POINTER(ctypes.c_double), - ctypes.c_int, - ctypes.c_int, - ctypes.POINTER(HarvestOption), - ctypes.POINTER(ctypes.c_double), - ctypes.POINTER(ctypes.c_double) - ] - - self.world_dll.Harvest.restype = None - self.world_dll.InitializeHarvestOption.argtypes = [ctypes.POINTER(HarvestOption)] - self.world_dll.InitializeHarvestOption.restype = None - self.world_dll.GetSamplesForHarvest.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_double] - self.world_dll.GetSamplesForHarvest.restype = ctypes.c_int - - option = HarvestOption() - self.world_dll.InitializeHarvestOption(ctypes.byref(option)) - - option.F0Floor = f0_floor - option.F0Ceil = f0_ceil - option.FramePeriod = frame_period - - f0_length = self.world_dll.GetSamplesForHarvest(fs, len(x), option.FramePeriod) - f0 = (ctypes.c_double * f0_length)() - tpos = (ctypes.c_double * f0_length)() - - self.world_dll.Harvest((ctypes.c_double * len(x))(*x), len(x), fs, ctypes.byref(option), tpos, f0) - return np.array(f0, dtype=np.float32), np.array(tpos, dtype=np.float32) - - def dio( - self, - x, - fs, - f0_floor=50, - f0_ceil=1100, - channels_in_octave=2, - frame_period=10, - speed=1, - allowed_range=0.1 - ): - self.world_dll.Dio.argtypes = [ - ctypes.POINTER(ctypes.c_double), - ctypes.c_int, - ctypes.c_int, - ctypes.POINTER(DioOption), - ctypes.POINTER(ctypes.c_double), - ctypes.POINTER(ctypes.c_double) - ] - - self.world_dll.Dio.restype = None - self.world_dll.InitializeDioOption.argtypes = [ctypes.POINTER(DioOption)] - self.world_dll.InitializeDioOption.restype = None - self.world_dll.GetSamplesForDIO.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_double] - self.world_dll.GetSamplesForDIO.restype = ctypes.c_int - - option = DioOption() - self.world_dll.InitializeDioOption(ctypes.byref(option)) - - option.F0Floor = f0_floor - option.F0Ceil = f0_ceil - option.ChannelsInOctave = channels_in_octave - option.FramePeriod = frame_period - option.Speed = speed - option.AllowedRange = allowed_range - - f0_length = self.world_dll.GetSamplesForDIO(fs, len(x), option.FramePeriod) - f0 = (ctypes.c_double * f0_length)() - tpos = (ctypes.c_double * f0_length)() - - self.world_dll.Dio((ctypes.c_double * len(x))(*x), len(x), fs, ctypes.byref(option), tpos, f0) - return np.array(f0, dtype=np.float32), np.array(tpos, dtype=np.float32) - - def stonemask( - self, - x, - fs, - tpos, - f0 - ): - self.world_dll.StoneMask.argtypes = [ - ctypes.POINTER(ctypes.c_double), - ctypes.c_int, - ctypes.c_int, - ctypes.POINTER(ctypes.c_double), - ctypes.POINTER(ctypes.c_double), - ctypes.c_int, - ctypes.POINTER(ctypes.c_double) - ] - - self.world_dll.StoneMask.restype = None - - out_f0 = (ctypes.c_double * len(f0))() - self.world_dll.StoneMask( - (ctypes.c_double * len(x))(*x), - len(x), - fs, - (ctypes.c_double * len(tpos))(*tpos), - (ctypes.c_double * len(f0))(*f0), - len(f0), - out_f0 - ) - - return np.array(out_f0, dtype=np.float32) \ No newline at end of file diff --git a/main/library/speaker_diarization/ECAPA_TDNN.py b/main/library/speaker_diarization/ECAPA_TDNN.py deleted file mode 100644 index 3b002d45e6f8d900dbe3ea9b683dc7ea983fe7e8..0000000000000000000000000000000000000000 --- a/main/library/speaker_diarization/ECAPA_TDNN.py +++ /dev/null @@ -1,583 +0,0 @@ -import math -import torch - -import torch.nn as nn -import torch.nn.functional as F - -def length_to_mask(length, max_len=None, dtype=None, device=None): - assert len(length.shape) == 1 - - if max_len is None: max_len = length.max().long().item() - - mask = torch.arange(max_len, device=length.device, dtype=length.dtype).expand(len(length), max_len) < length.unsqueeze(1) - - if dtype is None: dtype = length.dtype - if device is None: device = length.device - - return torch.as_tensor(mask, dtype=dtype, device=device) - -def get_padding_elem(L_in, stride, kernel_size, dilation): - if stride > 1: padding = [math.floor(kernel_size / 2), math.floor(kernel_size / 2)] - else: - L_out = (math.floor((L_in - dilation * (kernel_size - 1) - 1) / stride) + 1) - padding = [math.floor((L_in - L_out) / 2), math.floor((L_in - L_out) / 2)] - - return padding - -class _BatchNorm1d(nn.Module): - def __init__( - self, - input_shape=None, - input_size=None, - eps=1e-05, - momentum=0.1, - affine=True, - track_running_stats=True, - combine_batch_time=False, - skip_transpose=False - ): - super().__init__() - self.combine_batch_time = combine_batch_time - self.skip_transpose = skip_transpose - - if input_size is None and skip_transpose: input_size = input_shape[1] - elif input_size is None: input_size = input_shape[-1] - - self.norm = nn.BatchNorm1d( - input_size, - eps=eps, - momentum=momentum, - affine=affine, - track_running_stats=track_running_stats - ) - - def forward(self, x): - shape_or = x.shape - - if self.combine_batch_time: - x = ( - x.reshape(shape_or[0] * shape_or[1], shape_or[2]) - ) if x.ndim == 3 else ( - x.reshape(shape_or[0] * shape_or[1], shape_or[3], shape_or[2]) - ) - elif not self.skip_transpose: x = x.transpose(-1, 1) - - x_n = self.norm(x) - - if self.combine_batch_time: x_n = x_n.reshape(shape_or) - elif not self.skip_transpose: x_n = x_n.transpose(1, -1) - - return x_n - -class _Conv1d(nn.Module): - def __init__( - self, - out_channels, - kernel_size, - input_shape=None, - in_channels=None, - stride=1, - dilation=1, - padding="same", - groups=1, - bias=True, - padding_mode="reflect", - skip_transpose=False, - weight_norm=False, - conv_init=None, - default_padding=0 - ): - super().__init__() - self.kernel_size = kernel_size - self.stride = stride - self.dilation = dilation - self.padding = padding - self.padding_mode = padding_mode - self.unsqueeze = False - self.skip_transpose = skip_transpose - - if input_shape is None and in_channels is None: raise ValueError - if in_channels is None: in_channels = self._check_input_shape(input_shape) - - self.in_channels = in_channels - self.conv = nn.Conv1d( - in_channels, - out_channels, - self.kernel_size, - stride=self.stride, - dilation=self.dilation, - padding=default_padding, - groups=groups, - bias=bias - ) - - if conv_init == "kaiming": nn.init.kaiming_normal_(self.conv.weight) - elif conv_init == "zero": nn.init.zeros_(self.conv.weight) - elif conv_init == "normal": nn.init.normal_(self.conv.weight, std=1e-6) - - if weight_norm: self.conv = nn.utils.weight_norm(self.conv) - - def forward(self, x): - if not self.skip_transpose: x = x.transpose(1, -1) - if self.unsqueeze: x = x.unsqueeze(1) - - if self.padding == "same": - x = self._manage_padding( - x, - self.kernel_size, - self.dilation, - self.stride - ) - elif self.padding == "causal": - x = F.pad( - x, - ((self.kernel_size - 1) * self.dilation, 0) - ) - elif self.padding == "valid": pass - else: raise ValueError - - wx = self.conv(x) - - if self.unsqueeze: wx = wx.squeeze(1) - if not self.skip_transpose: wx = wx.transpose(1, -1) - - return wx - - def _manage_padding(self, x, kernel_size, dilation, stride): - return F.pad( - x, - get_padding_elem( - self.in_channels, - stride, - kernel_size, - dilation - ), - mode=self.padding_mode - ) - - def _check_input_shape(self, shape): - if len(shape) == 2: - self.unsqueeze = True - in_channels = 1 - elif self.skip_transpose: in_channels = shape[1] - elif len(shape) == 3: in_channels = shape[2] - else: raise ValueError - - if not self.padding == "valid" and self.kernel_size % 2 == 0: raise ValueError - return in_channels - - def remove_weight_norm(self): - self.conv = nn.utils.remove_weight_norm(self.conv) - -class Linear(torch.nn.Module): - def __init__( - self, - n_neurons, - input_shape=None, - input_size=None, - bias=True, - max_norm=None, - combine_dims=False - ): - super().__init__() - self.max_norm = max_norm - self.combine_dims = combine_dims - - if input_shape is None and input_size is None: raise ValueError - if input_size is None: - input_size = input_shape[-1] - if len(input_shape) == 4 and self.combine_dims: input_size = input_shape[2] * input_shape[3] - - self.w = nn.Linear(input_size, n_neurons, bias=bias) - - def forward(self, x): - if x.ndim == 4 and self.combine_dims: x = x.reshape(x.shape[0], x.shape[1], x.shape[2] * x.shape[3]) - if self.max_norm is not None: self.w.weight.data = torch.renorm(self.w.weight.data, p=2, dim=0, maxnorm=self.max_norm) - - return self.w(x) - -class Conv1d(_Conv1d): - def __init__( - self, - *args, - **kwargs - ): - super().__init__( - skip_transpose=True, - *args, - **kwargs - ) - -class BatchNorm1d(_BatchNorm1d): - def __init__( - self, - *args, - **kwargs - ): - super().__init__( - skip_transpose=True, - *args, - **kwargs - ) - -class TDNNBlock(nn.Module): - def __init__( - self, - in_channels, - out_channels, - kernel_size, - dilation, - activation=nn.ReLU, - groups=1, - dropout=0.0 - ): - super().__init__() - self.conv = Conv1d( - in_channels=in_channels, - out_channels=out_channels, - kernel_size=kernel_size, - dilation=dilation, - groups=groups - ) - - self.activation = activation() - self.norm = BatchNorm1d(input_size=out_channels) - self.dropout = nn.Dropout1d(p=dropout) - - def forward(self, x): - return self.dropout(self.norm(self.activation(self.conv(x)))) - -class Res2NetBlock(torch.nn.Module): - def __init__( - self, - in_channels, - out_channels, - scale=8, - kernel_size=3, - dilation=1, - dropout=0.0 - ): - super().__init__() - assert in_channels % scale == 0 - assert out_channels % scale == 0 - in_channel = in_channels // scale - hidden_channel = out_channels // scale - self.blocks = nn.ModuleList([ - TDNNBlock( - in_channel, - hidden_channel, - kernel_size=kernel_size, - dilation=dilation, - dropout=dropout - ) - for _ in range(scale - 1) - ]) - self.scale = scale - - def forward(self, x): - y = [] - - for i, x_i in enumerate(x.chunk(self.scale, dim=1)): - if i == 0: y_i = x_i - elif i == 1: y_i = self.blocks[i - 1](x_i) - else: y_i = self.blocks[i - 1](x_i + y_i) - - y.append(y_i) - - return torch.cat(y, dim=1) - -class SEBlock(nn.Module): - def __init__( - self, - in_channels, - se_channels, - out_channels - ): - super().__init__() - - self.conv1 = Conv1d( - in_channels=in_channels, - out_channels=se_channels, - kernel_size=1 - ) - self.relu = torch.nn.ReLU(inplace=True) - - self.conv2 = Conv1d( - in_channels=se_channels, - out_channels=out_channels, - kernel_size=1 - ) - self.sigmoid = torch.nn.Sigmoid() - - def forward(self, x, lengths=None): - L = x.shape[-1] - - if lengths is not None: - mask = length_to_mask( - lengths * L, - max_len=L, - device=x.device - ).unsqueeze(1) - - s = (x * mask).sum(dim=2, keepdim=True) / mask.sum(dim=2, keepdim=True) - else: s = x.mean(dim=2, keepdim=True) - - return self.sigmoid(self.conv2(self.relu(self.conv1(s)))) * x - -class AttentiveStatisticsPooling(nn.Module): - def __init__( - self, - channels, - attention_channels=128, - global_context=True - ): - super().__init__() - self.eps = 1e-12 - self.global_context = global_context - self.tdnn = ( - TDNNBlock( - channels * 3, - attention_channels, - 1, - 1 - ) - ) if global_context else ( - TDNNBlock( - channels, - attention_channels, - 1, - 1 - ) - ) - self.tanh = nn.Tanh() - self.conv = Conv1d( - in_channels=attention_channels, - out_channels=channels, - kernel_size=1 - ) - - def forward(self, x, lengths=None): - L = x.shape[-1] - - def _compute_statistics(x, m, dim=2, eps=self.eps): - mean = (m * x).sum(dim) - return mean, ((m * (x - mean.unsqueeze(dim)).pow(2)).sum(dim).clamp(eps)).sqrt() - - if lengths is None: lengths = torch.ones(x.shape[0], device=x.device) - mask = length_to_mask(lengths * L, max_len=L, device=x.device).unsqueeze(1) - - if self.global_context: - mean, std = _compute_statistics( - x, - mask / mask.sum(dim=2, keepdim=True).float() - ) - - attn = torch.cat([ - x, - mean.unsqueeze(2).repeat(1, 1, L), - std.unsqueeze(2).repeat(1, 1, L) - ], dim=1) - else: attn = x - - mean, std = _compute_statistics( - x, - F.softmax( - self.conv( - self.tanh(self.tdnn(attn)) - ).masked_fill(mask == 0, float("-inf")), - dim=2 - ) - ) - return torch.cat((mean, std), dim=1).unsqueeze(2) - -class SERes2NetBlock(nn.Module): - def __init__( - self, - in_channels, - out_channels, - res2net_scale=8, - se_channels=128, - kernel_size=1, - dilation=1, - activation=torch.nn.ReLU, - groups=1, - dropout=0.0 - ): - super().__init__() - self.out_channels = out_channels - self.tdnn1 = TDNNBlock( - in_channels, - out_channels, - kernel_size=1, - dilation=1, - activation=activation, - groups=groups, dropout=dropout - ) - self.res2net_block = Res2NetBlock( - out_channels, - out_channels, - res2net_scale, - kernel_size, - dilation - ) - self.tdnn2 = TDNNBlock( - out_channels, - out_channels, - kernel_size=1, - dilation=1, - activation=activation, - groups=groups, - dropout=dropout - ) - self.se_block = SEBlock( - out_channels, - se_channels, - out_channels - ) - - self.shortcut = None - if in_channels != out_channels: - self.shortcut = Conv1d( - in_channels=in_channels, - out_channels=out_channels, - kernel_size=1 - ) - - def forward(self, x, lengths=None): - residual = x - if self.shortcut: residual = self.shortcut(x) - - return self.se_block( - self.tdnn2(self.res2net_block(self.tdnn1(x))), - lengths - ) + residual - -class ECAPA_TDNN(torch.nn.Module): - def __init__( - self, - input_size, - device="cpu", - lin_neurons=192, - activation=torch.nn.ReLU, - channels=[512, 512, 512, 512, 1536], - kernel_sizes=[5, 3, 3, 3, 1], - dilations=[1, 2, 3, 4, 1], - attention_channels=128, - res2net_scale=8, - se_channels=128, - global_context=True, - groups=[1, 1, 1, 1, 1], - dropout=0.0 - ): - super().__init__() - assert len(channels) == len(kernel_sizes) - assert len(channels) == len(dilations) - - self.channels = channels - self.blocks = nn.ModuleList() - self.blocks.append( - TDNNBlock( - input_size, - channels[0], - kernel_sizes[0], - dilations[0], - activation, - groups[0], - dropout - ) - ) - - for i in range(1, len(channels) - 1): - self.blocks.append( - SERes2NetBlock( - channels[i - 1], - channels[i], - res2net_scale=res2net_scale, - se_channels=se_channels, - kernel_size=kernel_sizes[i], - dilation=dilations[i], - activation=activation, - groups=groups[i], - dropout=dropout - ) - ) - - self.mfa = TDNNBlock( - channels[-2] * (len(channels) - 2), - channels[-1], - kernel_sizes[-1], - dilations[-1], - activation, - groups=groups[-1], - dropout=dropout - ) - - self.asp = AttentiveStatisticsPooling( - channels[-1], - attention_channels=attention_channels, - global_context=global_context - ) - - self.asp_bn = BatchNorm1d( - input_size=channels[-1] * 2 - ) - - self.fc = Conv1d( - in_channels=channels[-1] * 2, - out_channels=lin_neurons, - kernel_size=1 - ) - - def forward(self, x, lengths=None): - x = x.transpose(1, 2) - xl = [] - - for layer in self.blocks: - try: - x = layer(x, lengths=lengths) - except TypeError: - x = layer(x) - - xl.append(x) - - return self.fc( - self.asp_bn(self.asp(self.mfa(torch.cat(xl[1:], dim=1)), lengths=lengths)) - ).transpose(1, 2) - -class Classifier(torch.nn.Module): - def __init__( - self, - input_size, - device="cpu", - lin_blocks=0, - lin_neurons=192, - out_neurons=1211 - ): - super().__init__() - self.blocks = nn.ModuleList() - - for _ in range(lin_blocks): - self.blocks.extend([ - _BatchNorm1d(input_size=input_size), - Linear(input_size=input_size, n_neurons=lin_neurons) - ]) - input_size = lin_neurons - - self.weight = nn.Parameter( - torch.FloatTensor( - out_neurons, - input_size, - device=device - ) - ) - - nn.init.xavier_uniform_(self.weight) - - def forward(self, x): - for layer in self.blocks: - x = layer(x) - - return F.linear( - F.normalize(x.squeeze(1)), - F.normalize(self.weight) - ).unsqueeze(1) \ No newline at end of file diff --git a/main/library/speaker_diarization/audio.py b/main/library/speaker_diarization/audio.py deleted file mode 100644 index 660ea8e86bd106debe9348fa38bff0324e5174a7..0000000000000000000000000000000000000000 --- a/main/library/speaker_diarization/audio.py +++ /dev/null @@ -1,187 +0,0 @@ -import os -import math -import torch -import random -import librosa -import torchaudio - -from io import IOBase - -class Audio: - @staticmethod - def power_normalize(waveform): - return waveform / (waveform.square().mean(dim=-1, keepdim=True).sqrt() + 1e-8) - - @staticmethod - def validate_file(file): - if isinstance(file, (str, os.PathLike)): - file = { - "audio": str(file), - "uri": os.path.splitext(os.path.basename(file))[0] - } - elif isinstance(file, IOBase): - return { - "audio": file, - "uri": "stream" - } - else: - raise ValueError - - if "waveform" in file: - waveform = file["waveform"] - if len(waveform.shape) != 2 or waveform.shape[0] > waveform.shape[1]: raise ValueError - - sample_rate = file.get("sample_rate", None) - if sample_rate is None: raise ValueError - - file.setdefault("uri", "waveform") - elif "audio" in file: - if isinstance(file["audio"], IOBase): return file - - path = os.path.abspath(file["audio"]) - file.setdefault("uri", os.path.splitext(os.path.basename(path))[0]) - else: - raise ValueError - - return file - - def __init__( - self, - sample_rate = None, - mono=None, - backend = None - ): - super().__init__() - self.sample_rate = sample_rate - self.mono = mono - - if not backend: backend = "soundfile" - self.backend = backend - - def downmix_and_resample(self, waveform, sample_rate): - num_channels = waveform.shape[0] - - if num_channels > 1: - if self.mono == "random": - channel = random.randint(0, num_channels - 1) - waveform = waveform[channel : channel + 1] - elif self.mono == "downmix": - waveform = waveform.mean(dim=0, keepdim=True) - - if (self.sample_rate is not None) and (self.sample_rate != sample_rate): - waveform = torchaudio.functional.resample( - waveform, - sample_rate, - self.sample_rate - ) - - sample_rate = self.sample_rate - - return waveform, sample_rate - - def get_num_samples(self, duration, sample_rate = None): - sample_rate = sample_rate or self.sample_rate - if sample_rate is None: raise ValueError - - return math.floor(duration * sample_rate) - - def __call__(self, file): - file = self.validate_file(file) - - if "waveform" in file: - waveform = file["waveform"] - sample_rate = file["sample_rate"] - elif "audio" in file: - try: - waveform, sample_rate = torchaudio.load(file["audio"], backend=self.backend) - except: - y, sample_rate = librosa.load(file["audio"], sr=None, mono=False) - waveform = torch.from_numpy(y) - - if isinstance(file["audio"], IOBase): file["audio"].seek(0) - - channel = file.get("channel", None) - if channel is not None: waveform = waveform[channel : channel + 1] - - return self.downmix_and_resample(waveform, sample_rate) - - def crop(self, file, segment, duration = None, mode="raise"): - file = self.validate_file(file) - - if "waveform" in file: - waveform = file["waveform"] - frames = waveform.shape[1] - sample_rate = file["sample_rate"] - elif "torchaudio.info" in file: - info = file["torchaudio.info"] - frames = info.num_frames - sample_rate = info.sample_rate - else: - info, sr = librosa.load(file["audio"], sr=None) - frames = info.shape[0] - sample_rate = sr - - channel = file.get("channel", None) - start_frame = math.floor(segment.start * sample_rate) - - if duration: - num_frames = math.floor(duration * sample_rate) - end_frame = start_frame + num_frames - else: - end_frame = math.floor(segment.end * sample_rate) - num_frames = end_frame - start_frame - - if mode == "raise": - if num_frames > frames: raise ValueError - - if end_frame > frames + math.ceil(0.001 * sample_rate): raise ValueError - else: - end_frame = min(end_frame, frames) - start_frame = end_frame - num_frames - - if start_frame < 0: raise ValueError - elif mode == "pad": - pad_start = -min(0, start_frame) - pad_end = max(end_frame, frames) - frames - - start_frame = max(0, start_frame) - end_frame = min(end_frame, frames) - - num_frames = end_frame - start_frame - - if "waveform" in file: - data = file["waveform"][:, start_frame:end_frame] - else: - try: - try: - data, _ = torchaudio.load( - file["audio"], - frame_offset=start_frame, - num_frames=num_frames, - backend=self.backend - ) - except: - y, _ = librosa.load( - file["audio"], - sr=sample_rate, - offset=start_frame / sample_rate, - duration=num_frames / sample_rate, - mono=False - ) - data = torch.from_numpy(y) - data = data.unsqueeze(0) if len(data.shape) == 1 else data - - if isinstance(file["audio"], IOBase): file["audio"].seek(0) - except RuntimeError: - if isinstance(file["audio"], IOBase): raise RuntimeError - - waveform, sample_rate = self.__call__(file) - data = waveform[:, start_frame:end_frame] - - file["waveform"] = waveform - file["sample_rate"] = sample_rate - - if channel is not None: data = data[channel : channel + 1, :] - if mode == "pad": data = torch.nn.functional.pad(data, (pad_start, pad_end)) - - return self.downmix_and_resample(data, sample_rate) \ No newline at end of file diff --git a/main/library/speaker_diarization/embedding.py b/main/library/speaker_diarization/embedding.py deleted file mode 100644 index 7fc0425dbdbb9823c34c2b1eb4674c00798fb323..0000000000000000000000000000000000000000 --- a/main/library/speaker_diarization/embedding.py +++ /dev/null @@ -1,96 +0,0 @@ -import os -import sys -import torch - -import numpy as np -import torch.nn.functional as F - -from functools import cached_property -from torch.nn.utils.rnn import pad_sequence - -sys.path.append(os.getcwd()) - -from main.library.speaker_diarization.speechbrain import EncoderClassifier - -class SpeechBrainPretrainedSpeakerEmbedding: - def __init__( - self, - embedding, - device = None - ): - super().__init__() - - self.embedding = embedding - self.device = device or torch.device("cpu") - self.classifier_ = EncoderClassifier.from_hparams( - source=self.embedding, - run_opts={"device": self.device} - ) - - @cached_property - def dimension(self): - *_, dimension = self.classifier_.encode_batch( - torch.rand(1, 16000).to(self.device) - ).shape - - return dimension - - @cached_property - def min_num_samples(self): - with torch.inference_mode(): - lower, upper = 2, round(0.5 * self.classifier_.audio_normalizer.sample_rate) - middle = (lower + upper) // 2 - - while lower + 1 < upper: - try: - _ = self.classifier_.encode_batch(torch.randn(1, middle).to(self.device)) - upper = middle - except RuntimeError: - lower = middle - - middle = (lower + upper) // 2 - - return upper - - def __call__(self, waveforms, masks = None): - batch_size, num_channels, num_samples = waveforms.shape - assert num_channels == 1 - - waveforms = waveforms.squeeze(dim=1) - - if masks is None: - signals = waveforms.squeeze(dim=1) - wav_lens = signals.shape[1] * torch.ones(batch_size) - else: - batch_size_masks, _ = masks.shape - assert batch_size == batch_size_masks - - imasks = F.interpolate( - masks.unsqueeze(dim=1), - size=num_samples, - mode="nearest" - ).squeeze(dim=1) > 0.5 - - signals = pad_sequence([ - waveform[imask].contiguous() - for waveform, imask in zip(waveforms, imasks) - ], batch_first=True) - - wav_lens = imasks.sum(dim=1) - - max_len = wav_lens.max() - if max_len < self.min_num_samples: return np.nan * np.zeros((batch_size, self.dimension)) - - too_short = wav_lens < self.min_num_samples - wav_lens = wav_lens / max_len - wav_lens[too_short] = 1.0 - - embeddings = ( - self.classifier_.encode_batch( - signals, - wav_lens=wav_lens - ).squeeze(dim=1).cpu().numpy() - ) - - embeddings[too_short.cpu().numpy()] = np.nan - return embeddings \ No newline at end of file diff --git a/main/library/speaker_diarization/encoder.py b/main/library/speaker_diarization/encoder.py deleted file mode 100644 index 5688b86484309122c89be7ccf233f3fa2f6a96c5..0000000000000000000000000000000000000000 --- a/main/library/speaker_diarization/encoder.py +++ /dev/null @@ -1,275 +0,0 @@ -import os -import sys -import ast -import torch -import itertools -import collections - -sys.path.append(os.getcwd()) - -from main.library.speaker_diarization.speechbrain import if_main_process, ddp_barrier -from main.library.speaker_diarization.features import register_checkpoint_hooks, mark_as_saver, mark_as_loader - -@register_checkpoint_hooks -class CategoricalEncoder: - VALUE_SEPARATOR = " => " - EXTRAS_SEPARATOR = "================\n" - - def __init__( - self, - starting_index=0, - **special_labels - ): - self.lab2ind = {} - self.ind2lab = {} - self.starting_index = starting_index - self.handle_special_labels(special_labels) - - def handle_special_labels(self, special_labels): - if "unk_label" in special_labels: self.add_unk(special_labels["unk_label"]) - - def __len__(self): - return len(self.lab2ind) - - @classmethod - def from_saved(cls, path): - obj = cls() - obj.load(path) - return obj - - def update_from_iterable(self, iterable, sequence_input=False): - label_iterator = itertools.chain.from_iterable(iterable) if sequence_input else iter(iterable) - for label in label_iterator: - self.ensure_label(label) - - def update_from_didataset(self, didataset, output_key, sequence_input=False): - with didataset.output_keys_as([output_key]): - self.update_from_iterable( - (data_point[output_key] for data_point in didataset), - sequence_input=sequence_input - ) - - def limited_labelset_from_iterable( - self, - iterable, - sequence_input=False, - n_most_common=None, - min_count=1 - ): - label_iterator = itertools.chain.from_iterable(iterable) if sequence_input else iter(iterable) - counts = collections.Counter(label_iterator) - - for label, count in counts.most_common(n_most_common): - if count < min_count: break - self.add_label(label) - - return counts - - def load_or_create( - self, - path, - from_iterables=[], - from_didatasets=[], - sequence_input=False, - output_key=None, - special_labels={} - ): - try: - if if_main_process(): - if not self.load_if_possible(path): - for iterable in from_iterables: - self.update_from_iterable(iterable, sequence_input) - - for didataset in from_didatasets: - if output_key is None: raise ValueError - self.update_from_didataset(didataset, output_key, sequence_input) - - self.handle_special_labels(special_labels) - self.save(path) - finally: - ddp_barrier() - self.load(path) - - def add_label(self, label): - if label in self.lab2ind: raise KeyError - index = self._next_index() - - self.lab2ind[label] = index - self.ind2lab[index] = label - - return index - - def ensure_label(self, label): - if label in self.lab2ind: return self.lab2ind[label] - else: return self.add_label(label) - - def insert_label(self, label, index): - if label in self.lab2ind: raise KeyError - else: self.enforce_label(label, index) - - def enforce_label(self, label, index): - index = int(index) - - if label in self.lab2ind: - if index == self.lab2ind[label]: return - else: del self.ind2lab[self.lab2ind[label]] - - if index in self.ind2lab: - saved_label = self.ind2lab[index] - moving_other = True - else: moving_other = False - - self.lab2ind[label] = index - self.ind2lab[index] = label - - if moving_other: - new_index = self._next_index() - self.lab2ind[saved_label] = new_index - self.ind2lab[new_index] = saved_label - - def add_unk(self, unk_label=""): - self.unk_label = unk_label - return self.add_label(unk_label) - - def _next_index(self): - index = self.starting_index - while index in self.ind2lab: - index += 1 - - return index - - def is_continuous(self): - indices = sorted(self.ind2lab.keys()) - return self.starting_index in indices and all(j - i == 1 for i, j in zip(indices[:-1], indices[1:])) - - def encode_label(self, label, allow_unk=True): - self._assert_len() - - try: - return self.lab2ind[label] - except KeyError: - if hasattr(self, "unk_label") and allow_unk: return self.lab2ind[self.unk_label] - elif hasattr(self, "unk_label") and not allow_unk: raise KeyError - elif not hasattr(self, "unk_label") and allow_unk: raise KeyError - else: raise KeyError - - def encode_label_torch(self, label, allow_unk=True): - return torch.LongTensor([self.encode_label(label, allow_unk)]) - - def encode_sequence(self, sequence, allow_unk=True): - self._assert_len() - return [self.encode_label(label, allow_unk) for label in sequence] - - def encode_sequence_torch(self, sequence, allow_unk=True): - return torch.LongTensor([self.encode_label(label, allow_unk) for label in sequence]) - - def decode_torch(self, x): - self._assert_len() - decoded = [] - - if x.ndim == 1: - for element in x: - decoded.append(self.ind2lab[int(element)]) - else: - for subtensor in x: - decoded.append(self.decode_torch(subtensor)) - - return decoded - - def decode_ndim(self, x): - self._assert_len() - try: - decoded = [] - for subtensor in x: - decoded.append(self.decode_ndim(subtensor)) - - return decoded - except TypeError: - return self.ind2lab[int(x)] - - @mark_as_saver - def save(self, path): - self._save_literal(path, self.lab2ind, self._get_extras()) - - def load(self, path): - lab2ind, ind2lab, extras = self._load_literal(path) - self.lab2ind = lab2ind - self.ind2lab = ind2lab - self._set_extras(extras) - - @mark_as_loader - def load_if_possible(self, path, end_of_epoch=False): - del end_of_epoch - - try: - self.load(path) - except FileNotFoundError: - return False - except (ValueError, SyntaxError): - return False - - return True - - def expect_len(self, expected_len): - self.expected_len = expected_len - - def ignore_len(self): - self.expected_len = None - - def _assert_len(self): - if hasattr(self, "expected_len"): - if self.expected_len is None: return - if len(self) != self.expected_len: raise RuntimeError - else: - self.ignore_len() - return - - def _get_extras(self): - extras = {"starting_index": self.starting_index} - if hasattr(self, "unk_label"): extras["unk_label"] = self.unk_label - - return extras - - def _set_extras(self, extras): - if "unk_label" in extras: self.unk_label = extras["unk_label"] - self.starting_index = extras["starting_index"] - - @staticmethod - def _save_literal(path, lab2ind, extras): - with open(path, "w", encoding="utf-8") as f: - for label, ind in lab2ind.items(): - f.write(repr(label) + CategoricalEncoder.VALUE_SEPARATOR + str(ind) + "\n") - - f.write(CategoricalEncoder.EXTRAS_SEPARATOR) - - for key, value in extras.items(): - f.write(repr(key) + CategoricalEncoder.VALUE_SEPARATOR + repr(value) + "\n") - - f.flush() - - @staticmethod - def _load_literal(path): - lab2ind, ind2lab, extras = {}, {}, {} - - with open(path, encoding="utf-8") as f: - for line in f: - if line == CategoricalEncoder.EXTRAS_SEPARATOR: break - - literal, ind = line.strip().split( - CategoricalEncoder.VALUE_SEPARATOR, - maxsplit=1 - ) - - label = ast.literal_eval(literal) - lab2ind[label] = int(ind) - ind2lab[ind] = label - - for line in f: - literal_key, literal_value = line.strip().split( - CategoricalEncoder.VALUE_SEPARATOR, - maxsplit=1 - ) - - extras[ast.literal_eval(literal_key)] = ast.literal_eval(literal_value) - - return lab2ind, ind2lab, extras \ No newline at end of file diff --git a/main/library/speaker_diarization/features.py b/main/library/speaker_diarization/features.py deleted file mode 100644 index 799c1f4b4ce48ced1b59ebefd895d4a803319ada..0000000000000000000000000000000000000000 --- a/main/library/speaker_diarization/features.py +++ /dev/null @@ -1,738 +0,0 @@ -import os -import sys -import math -import torch -import inspect -import functools - -sys.path.append(os.getcwd()) - -from main.app.variables import config -from main.library.speaker_diarization.speechbrain import MAIN_PROC_ONLY, is_distributed_initialized, main_process_only - -WEAKREF_MARKER = "WEAKREF" - -KEYS_MAPPING = { - ".mutihead_attn": ".multihead_attn", - ".convs_intermedite": ".convs_intermediate" -} - -def map_old_state_dict_weights(state_dict, mapping): - for replacement_old, replacement_new in mapping.items(): - for old_key in list(state_dict.keys()): - if replacement_old in old_key: - state_dict[ - old_key.replace(replacement_old, replacement_new) - ] = state_dict.pop(old_key) - - return state_dict - -def hook_on_loading_state_dict_checkpoint(state_dict): - return map_old_state_dict_weights( - state_dict, - KEYS_MAPPING - ) - -def torch_patched_state_dict_load(path, device="cpu"): - return hook_on_loading_state_dict_checkpoint( - torch.load( - path, - map_location=device, - weights_only=True - ) - ) - -@main_process_only -def torch_save(obj, path): - state_dict = obj.state_dict() - torch.save(state_dict, path) - -def torch_recovery(obj, path, end_of_epoch): - del end_of_epoch - - state_dict = torch_patched_state_dict_load(path, "cpu") - - try: - obj.load_state_dict(state_dict, strict=True) - except TypeError: - obj.load_state_dict(state_dict) - -def torch_parameter_transfer(obj, path): - incompatible_keys = obj.load_state_dict( - torch_patched_state_dict_load(path, "cpu"), - strict=False - ) - - for missing_key in incompatible_keys.missing_keys: - pass - for unexpected_key in incompatible_keys.unexpected_keys: - pass - -def _cycliclrsaver(obj, path): - state_dict = obj.state_dict() - if state_dict.get("_scale_fn_ref") is not None: state_dict["_scale_fn_ref"] = WEAKREF_MARKER - - torch.save(state_dict, path) - -def _cycliclrloader(obj, path, end_of_epoch): - del end_of_epoch - - try: - obj.load_state_dict( - torch.load(path, map_location="cpu", weights_only=True), - strict=True - ) - except TypeError: - obj.load_state_dict( - torch.load(path, map_location="cpu", weights_only=True) - ) - -DEFAULT_LOAD_HOOKS = { - torch.nn.Module: torch_recovery, - torch.optim.Optimizer: torch_recovery, - torch.optim.lr_scheduler.ReduceLROnPlateau: torch_recovery, - torch.cuda.amp.grad_scaler.GradScaler: torch_recovery -} - -DEFAULT_SAVE_HOOKS = { - torch.nn.Module: torch_save, - torch.optim.Optimizer: torch_save, - torch.optim.lr_scheduler.ReduceLROnPlateau: torch_save, - torch.cuda.amp.grad_scaler.GradScaler: torch_save -} - -DEFAULT_LOAD_HOOKS[ - torch.optim.lr_scheduler.LRScheduler -] = torch_recovery - -DEFAULT_SAVE_HOOKS[ - torch.optim.lr_scheduler.LRScheduler -] = torch_save - -DEFAULT_TRANSFER_HOOKS = { - torch.nn.Module: torch_parameter_transfer -} - -DEFAULT_SAVE_HOOKS[ - torch.optim.lr_scheduler.CyclicLR -] = _cycliclrsaver - -DEFAULT_LOAD_HOOKS[ - torch.optim.lr_scheduler.CyclicLR -] = _cycliclrloader - -def register_checkpoint_hooks(cls, save_on_main_only=True): - global DEFAULT_LOAD_HOOKS, DEFAULT_SAVE_HOOKS, DEFAULT_TRANSFER_HOOKS - - for name, method in cls.__dict__.items(): - if hasattr(method, "_speechbrain_saver"): DEFAULT_SAVE_HOOKS[cls] = main_process_only(method) if save_on_main_only else method - if hasattr(method, "_speechbrain_loader"): DEFAULT_LOAD_HOOKS[cls] = method - if hasattr(method, "_speechbrain_transfer"): DEFAULT_TRANSFER_HOOKS[cls] = method - - return cls - -def mark_as_saver(method): - sig = inspect.signature(method) - - try: - sig.bind(object(), "testpath") - except TypeError: - raise TypeError - - method._speechbrain_saver = True - return method - -def mark_as_transfer(method): - sig = inspect.signature(method) - - try: - sig.bind(object(), "testpath") - except TypeError: - raise TypeError - - method._speechbrain_transfer = True - return method - -def mark_as_loader(method): - sig = inspect.signature(method) - - try: - sig.bind(object(), "testpath", True) - except TypeError: - raise TypeError - - method._speechbrain_loader = True - return method - -def ddp_all_reduce(communication_object, reduce_op): - if MAIN_PROC_ONLY >= 1 or not is_distributed_initialized(): return communication_object - torch.distributed.all_reduce(communication_object, op=reduce_op) - - return communication_object - -def fwd_default_precision(fwd = None, cast_inputs = torch.float32): - if fwd is None: return functools.partial(fwd_default_precision, cast_inputs=cast_inputs) - - wrapped_fwd = torch.amp.custom_fwd(fwd, cast_inputs=cast_inputs, device_type=config.device if config.device.startswith(("cuda", "xpu")) else "cpu") - - @functools.wraps(fwd) - def wrapper(*args, force_allow_autocast = False, **kwargs): - return fwd(*args, **kwargs) if force_allow_autocast else wrapped_fwd(*args, **kwargs) - - return wrapper - -def spectral_magnitude(stft, power = 1, log = False, eps = 1e-14): - spectr = stft.pow(2).sum(-1) - - if power < 1: spectr = spectr + eps - spectr = spectr.pow(power) - - if log: return (spectr + eps).log() - return spectr - -class Filterbank(torch.nn.Module): - def __init__( - self, - n_mels=40, - log_mel=True, - filter_shape="triangular", - f_min=0, - f_max=8000, - n_fft=400, - sample_rate=16000, - power_spectrogram=2, - amin=1e-10, - ref_value=1.0, - top_db=80.0, - param_change_factor=1.0, - param_rand_factor=0.0, - freeze=True - ): - super().__init__() - self.n_mels = n_mels - self.log_mel = log_mel - self.filter_shape = filter_shape - self.f_min = f_min - self.f_max = f_max - self.n_fft = n_fft - self.sample_rate = sample_rate - self.power_spectrogram = power_spectrogram - self.amin = amin - self.ref_value = ref_value - self.top_db = top_db - self.freeze = freeze - self.n_stft = self.n_fft // 2 + 1 - self.db_multiplier = math.log10(max(self.amin, self.ref_value)) - self.device_inp = torch.device("cpu") - self.param_change_factor = param_change_factor - self.param_rand_factor = param_rand_factor - self.multiplier = 10 if self.power_spectrogram == 2 else 20 - - hz = self._to_hz(torch.linspace(self._to_mel(self.f_min), self._to_mel(self.f_max), self.n_mels + 2)) - - band = hz[1:] - hz[:-1] - self.band = band[:-1] - self.f_central = hz[1:-1] - - if not self.freeze: - self.f_central = torch.nn.Parameter(self.f_central / (self.sample_rate * self.param_change_factor)) - self.band = torch.nn.Parameter(self.band / (self.sample_rate * self.param_change_factor)) - - self.all_freqs_mat = torch.linspace(0, self.sample_rate // 2, self.n_stft).repeat(self.f_central.shape[0], 1) - - def forward(self, spectrogram): - f_central_mat = self.f_central.repeat(self.all_freqs_mat.shape[1], 1).transpose(0, 1) - band_mat = self.band.repeat(self.all_freqs_mat.shape[1], 1).transpose(0, 1) - - if not self.freeze: - f_central_mat = f_central_mat * (self.sample_rate * self.param_change_factor * self.param_change_factor) - band_mat = band_mat * (self.sample_rate * self.param_change_factor * self.param_change_factor) - elif self.param_rand_factor != 0 and self.training: - rand_change = (1.0 + torch.rand(2) * 2 * self.param_rand_factor - self.param_rand_factor) - f_central_mat = f_central_mat * rand_change[0] - band_mat = band_mat * rand_change[1] - - fbank_matrix = self._create_fbank_matrix(f_central_mat, band_mat).to(spectrogram.device) - sp_shape = spectrogram.shape - if len(sp_shape) == 4: spectrogram = spectrogram.permute(0, 3, 1, 2).reshape(sp_shape[0] * sp_shape[3], sp_shape[1], sp_shape[2]) - - fbanks = spectrogram @ fbank_matrix - if self.log_mel: fbanks = self._amplitude_to_DB(fbanks) - - if len(sp_shape) == 4: - fb_shape = fbanks.shape - fbanks = fbanks.reshape(sp_shape[0], sp_shape[3], fb_shape[1], fb_shape[2]).permute(0, 2, 3, 1) - - return fbanks - - @staticmethod - def _to_mel(hz): - return 2595 * math.log10(1 + hz / 700) - - @staticmethod - def _to_hz(mel): - return 700 * (10 ** (mel / 2595) - 1) - - def _triangular_filters(self, all_freqs, f_central, band): - slope = (all_freqs - f_central) / band - return torch.zeros(1, device=self.device_inp).max((slope + 1.0).min(-slope + 1.0)).transpose(0, 1) - - def _rectangular_filters(self, all_freqs, f_central, band): - left_side = right_size = all_freqs.ge(f_central - band) - right_size = all_freqs.le(f_central + band) - - return (left_side * right_size).float().transpose(0, 1) - - def _gaussian_filters(self, all_freqs, f_central, band, smooth_factor=torch.tensor(2)): - return (-0.5 * ((all_freqs - f_central) / (band / smooth_factor)) ** 2).exp().transpose(0, 1) - - def _create_fbank_matrix(self, f_central_mat, band_mat): - if self.filter_shape == "triangular": fbank_matrix = self._triangular_filters(self.all_freqs_mat, f_central_mat, band_mat) - elif self.filter_shape == "rectangular": fbank_matrix = self._rectangular_filters(self.all_freqs_mat, f_central_mat, band_mat) - else: fbank_matrix = self._gaussian_filters(self.all_freqs_mat, f_central_mat, band_mat) - - return fbank_matrix - - def _amplitude_to_DB(self, x): - x_db = self.multiplier * x.clamp(min=self.amin).log10() - x_db -= self.multiplier * self.db_multiplier - - return x_db.max((x_db.amax(dim=(-2, -1)) - self.top_db).view(x_db.shape[0], 1, 1)) - -class ContextWindow(torch.nn.Module): - def __init__( - self, - left_frames=0, - right_frames=0 - ): - super().__init__() - self.left_frames = left_frames - self.right_frames = right_frames - self.context_len = self.left_frames + self.right_frames + 1 - self.kernel_len = 2 * max(self.left_frames, self.right_frames) + 1 - self.kernel = torch.eye(self.context_len, self.kernel_len) - - if self.right_frames > self.left_frames: self.kernel = torch.roll(self.kernel, self.right_frames - self.left_frames, 1) - self.first_call = True - - def forward(self, x): - x = x.transpose(1, 2) - if self.first_call: - self.first_call = False - self.kernel = (self.kernel.repeat(x.shape[1], 1, 1).view(x.shape[1] * self.context_len, self.kernel_len).unsqueeze(1)) - - or_shape = x.shape - if len(or_shape) == 4: x = x.reshape(or_shape[0] * or_shape[2], or_shape[1], or_shape[3]) - - cw_x = torch.nn.functional.conv1d( - x, - self.kernel.to(x.device), - groups=x.shape[1], - padding=max(self.left_frames, self.right_frames) - ) - - if len(or_shape) == 4: cw_x = cw_x.reshape(or_shape[0], cw_x.shape[1], or_shape[2], cw_x.shape[-1]) - return cw_x.transpose(1, 2) - -class FilterProperties: - def __init__(self, window_size = 0, stride = 1, dilation = 1, causal = False): - self.window_size = window_size - self.stride = stride - self.dilation = dilation - self.causal = causal - - def __post_init__(self): - assert self.window_size > 0 - assert self.stride > 0 - assert (self.dilation > 0) - - @staticmethod - def pointwise_filter(): - return FilterProperties(window_size=1, stride=1) - - def get_effective_size(self): - return 1 + ((self.window_size - 1) * self.dilation) - - def get_convolution_padding(self): - if self.window_size % 2 == 0: raise ValueError - if self.causal: return self.get_effective_size() - 1 - - return (self.get_effective_size() - 1) // 2 - - def get_noncausal_equivalent(self): - if not self.causal: return self - - return FilterProperties( - window_size=(self.window_size - 1) * 2 + 1, - stride=self.stride, - dilation=self.dilation, - causal=False - ) - - def with_on_top(self, other, allow_approximate=True): - self_size = self.window_size - - if other.window_size % 2 == 0: - if allow_approximate: other_size = other.window_size + 1 - else: raise ValueError - else: other_size = other.window_size - - if (self.causal or other.causal) and not (self.causal and other.causal): - if allow_approximate: return self.get_noncausal_equivalent().with_on_top(other.get_noncausal_equivalent()) - else: raise ValueError - - return FilterProperties(self_size + (self.stride * (other_size - 1)), self.stride * other.stride, self.dilation * other.dilation, self.causal) - -class STFT(torch.nn.Module): - def __init__( - self, - sample_rate, - win_length=25, - hop_length=10, - n_fft=400, - window_fn=torch.hamming_window, - normalized_stft=False, - center=True, - pad_mode="constant", - onesided=True - ): - super().__init__() - self.sample_rate = sample_rate - self.win_length = win_length - self.hop_length = hop_length - self.n_fft = n_fft - self.normalized_stft = normalized_stft - self.center = center - self.pad_mode = pad_mode - self.onesided = onesided - self.win_length = int(round((self.sample_rate / 1000.0) * self.win_length)) - self.hop_length = int(round((self.sample_rate / 1000.0) * self.hop_length)) - self.window = window_fn(self.win_length) - - def forward(self, x): - or_shape = x.shape - if len(or_shape) == 3: x = x.transpose(1, 2).reshape(or_shape[0] * or_shape[2], or_shape[1]) - - device = x.device - if str(device) not in ["cuda", "xpu", "cpu"]: x = x.cpu() - - stft = torch.view_as_real( - torch.stft( - x, - n_fft=self.n_fft, - hop_length=self.hop_length, - win_length=self.win_length, - window=self.window.to(x.device), - center=self.center, - pad_mode=self.pad_mode, - normalized=self.normalized_stft, - onesided=self.onesided, - return_complex=True - ) - ) - - stft = ( - stft.reshape( - or_shape[0], - or_shape[2], - stft.shape[1], - stft.shape[2], - stft.shape[3] - ).permute(0, 3, 2, 4, 1) - ) if len(or_shape) == 3 else ( - stft.transpose(2, 1) - ) - - return stft.to(device) - - def get_filter_properties(self): - if not self.center: raise ValueError - - return FilterProperties( - window_size=self.win_length, - stride=self.hop_length - ) - -class Deltas(torch.nn.Module): - def __init__( - self, - input_size, - window_length=5 - ): - super().__init__() - self.n = (window_length - 1) // 2 - self.denom = self.n * (self.n + 1) * (2 * self.n + 1) / 3 - self.register_buffer("kernel", torch.arange(-self.n, self.n + 1, dtype=torch.float32).repeat(input_size, 1, 1)) - - def forward(self, x): - x = x.transpose(1, 2).transpose(2, -1) - or_shape = x.shape - - if len(or_shape) == 4: - x = x.reshape( - or_shape[0] * or_shape[2], - or_shape[1], - or_shape[3] - ) - - x = torch.nn.functional.pad( - x, - (self.n, self.n), - mode="replicate" - ) - - delta_coeff = ( - torch.nn.functional.conv1d( - x, - self.kernel.to(x.device), - groups=x.shape[1] - ) / self.denom - ) - - if len(or_shape) == 4: - delta_coeff = delta_coeff.reshape( - or_shape[0], - or_shape[1], - or_shape[2], - or_shape[3] - ) - - return delta_coeff.transpose(1, -1).transpose(2, -1) - -class Fbank(torch.nn.Module): - def __init__( - self, - deltas=False, - context=False, - requires_grad=False, - sample_rate=16000, - f_min=0, - f_max=None, - n_fft=400, - n_mels=40, - filter_shape="triangular", - param_change_factor=1.0, - param_rand_factor=0.0, - left_frames=5, - right_frames=5, - win_length=25, - hop_length=10 - ): - super().__init__() - self.deltas = deltas - self.context = context - self.requires_grad = requires_grad - if f_max is None: f_max = sample_rate / 2 - - self.compute_STFT = STFT( - sample_rate=sample_rate, - n_fft=n_fft, - win_length=win_length, - hop_length=hop_length - ) - self.compute_fbanks = Filterbank( - sample_rate=sample_rate, - n_fft=n_fft, - n_mels=n_mels, - f_min=f_min, - f_max=f_max, - freeze=not requires_grad, - filter_shape=filter_shape, - param_change_factor=param_change_factor, - param_rand_factor=param_rand_factor - ) - self.compute_deltas = Deltas( - input_size=n_mels - ) - self.context_window = ContextWindow( - left_frames=left_frames, - right_frames=right_frames - ) - - @fwd_default_precision(cast_inputs=torch.float32) - def forward(self, wav): - fbanks = self.compute_fbanks( - spectral_magnitude( - self.compute_STFT(wav) - ) - ) - - if self.deltas: - delta1 = self.compute_deltas(fbanks) - fbanks = torch.cat([ - fbanks, delta1, - self.compute_deltas(delta1) - ], dim=2) - - if self.context: fbanks = self.context_window(fbanks) - return fbanks - - def get_filter_properties(self): - return self.compute_STFT.get_filter_properties() - -@register_checkpoint_hooks -class InputNormalization(torch.nn.Module): - def __init__( - self, - mean_norm=True, - std_norm=True, - norm_type="global", - avg_factor=None, - requires_grad=False, - update_until_epoch=3 - ): - super().__init__() - self.mean_norm = mean_norm - self.std_norm = std_norm - self.norm_type = norm_type - self.avg_factor = avg_factor - self.requires_grad = requires_grad - self.glob_mean = torch.tensor([0]) - self.glob_std = torch.tensor([0]) - self.spk_dict_mean = {} - self.spk_dict_std = {} - self.spk_dict_count = {} - self.weight = 1.0 - self.count = 0 - self.eps = 1e-10 - self.update_until_epoch = update_until_epoch - - def forward(self, x, lengths, spk_ids = torch.tensor([]), epoch=0): - N_batches = x.shape[0] - current_means, current_stds = [], [] - - if ( - self.norm_type == "sentence" or self.norm_type == "speaker" - ): - out = torch.empty_like(x) - - for snt_id in range(N_batches): - actual_size = torch.round(lengths[snt_id] * x.shape[1]).int() - current_mean, current_std = self._compute_current_stats(x[snt_id, 0:actual_size, ...]) - - current_means.append(current_mean) - current_stds.append(current_std) - - if self.norm_type == "sentence": out[snt_id] = (x[snt_id] - current_mean.data) / current_std.data - - if self.norm_type == "speaker": - spk_id = int(spk_ids[snt_id][0]) - - if self.training: - if spk_id not in self.spk_dict_mean: - self.spk_dict_mean[spk_id] = current_mean - self.spk_dict_std[spk_id] = current_std - self.spk_dict_count[spk_id] = 1 - else: - self.spk_dict_count[spk_id] = (self.spk_dict_count[spk_id] + 1) - self.weight = (1 / self.spk_dict_count[spk_id]) if self.avg_factor is None else self.avg_factor - - self.spk_dict_mean[spk_id] = (1 - self.weight) * self.spk_dict_mean[spk_id].to(current_mean) + self.weight * current_mean - self.spk_dict_std[spk_id] = (1 - self.weight) * self.spk_dict_std[spk_id].to(current_std) + self.weight * current_std - - self.spk_dict_mean[spk_id].detach() - self.spk_dict_std[spk_id].detach() - - speaker_mean = self.spk_dict_mean[spk_id].data - speaker_std = self.spk_dict_std[spk_id].data - else: - if spk_id in self.spk_dict_mean: - speaker_mean = self.spk_dict_mean[spk_id].data - speaker_std = self.spk_dict_std[spk_id].data - else: - speaker_mean = current_mean.data - speaker_std = current_std.data - - out[snt_id] = (x[snt_id] - speaker_mean) / speaker_std - - if self.norm_type == "batch" or self.norm_type == "global": - current_mean = ddp_all_reduce( - torch.stack(current_means).mean(dim=0), - torch.distributed.ReduceOp.AVG - ) - - current_std = ddp_all_reduce( - torch.stack(current_stds).mean(dim=0), - torch.distributed.ReduceOp.AVG - ) - - if self.norm_type == "batch": out = (x - current_mean.data) / (current_std.data) - - if self.norm_type == "global": - if self.training: - if self.count == 0: - self.glob_mean = current_mean - self.glob_std = current_std - elif epoch is None or epoch < self.update_until_epoch: - self.weight = (1 / (self.count + 1)) if self.avg_factor is None else self.avg_factor - self.glob_mean = (1 - self.weight) * self.glob_mean.to(current_mean) + self.weight * current_mean - self.glob_std = (1 - self.weight) * self.glob_std.to(current_std) + self.weight * current_std - - self.glob_mean.detach() - self.glob_std.detach() - self.count = self.count + 1 - - out = (x - self.glob_mean.data.to(x)) / (self.glob_std.data.to(x)) - - return out - - def _compute_current_stats(self, x): - current_std = x.std(dim=0).detach().data if self.std_norm else torch.tensor([1.0], device=x.device) - return ( - x.mean(dim=0).detach().data if self.mean_norm else torch.tensor([0.0], device=x.device), - current_std.max(self.eps * torch.ones_like(current_std)) - ) - - def _statistics_dict(self): - state = {} - state["count"] = self.count - state["glob_mean"] = self.glob_mean - state["glob_std"] = self.glob_std - state["spk_dict_mean"] = self.spk_dict_mean - state["spk_dict_std"] = self.spk_dict_std - state["spk_dict_count"] = self.spk_dict_count - - return state - - def _load_statistics_dict(self, state): - self.count = state["count"] - - if isinstance(state["glob_mean"], int): - self.glob_mean = state["glob_mean"] - self.glob_std = state["glob_std"] - else: - self.glob_mean = state["glob_mean"] - self.glob_std = state["glob_std"] - - self.spk_dict_mean = {} - for spk in state["spk_dict_mean"]: - self.spk_dict_mean[spk] = state["spk_dict_mean"][spk] - - self.spk_dict_std = {} - for spk in state["spk_dict_std"]: - self.spk_dict_std[spk] = state["spk_dict_std"][spk] - - self.spk_dict_count = state["spk_dict_count"] - return state - - def to(self, device): - self = super(InputNormalization, self).to(device) - self.glob_mean = self.glob_mean.to(device) - self.glob_std = self.glob_std.to(device) - - for spk in self.spk_dict_mean: - self.spk_dict_mean[spk] = self.spk_dict_mean[spk].to(device) - self.spk_dict_std[spk] = self.spk_dict_std[spk].to(device) - - return self - - @mark_as_saver - def _save(self, path): - torch.save(self._statistics_dict(), path) - - @mark_as_transfer - @mark_as_loader - def _load(self, path, end_of_epoch=False): - del end_of_epoch - stats = torch.load(path, map_location="cpu", weights_only=True) - self._load_statistics_dict(stats) \ No newline at end of file diff --git a/main/library/speaker_diarization/parameter_transfer.py b/main/library/speaker_diarization/parameter_transfer.py deleted file mode 100644 index ec93fc3ade67c9769f504260bd269718da2eff3a..0000000000000000000000000000000000000000 --- a/main/library/speaker_diarization/parameter_transfer.py +++ /dev/null @@ -1,136 +0,0 @@ -import os -import sys -import inspect - -sys.path.append(os.getcwd()) - -from main.library.speaker_diarization.speechbrain import fetch, run_on_main -from main.library.speaker_diarization.features import DEFAULT_TRANSFER_HOOKS, DEFAULT_LOAD_HOOKS - -def get_default_hook(obj, default_hooks): - for cls in inspect.getmro(type(obj)): - if cls in default_hooks: return default_hooks[cls] - - return None - -class Pretrainer: - def __init__( - self, - loadables=None, - paths=None, - custom_hooks=None, - conditions=None - ): - self.loadables = {} - - if loadables is not None: self.add_loadables(loadables) - self.paths = {} - - if paths is not None: self.add_paths(paths) - self.custom_hooks = {} - - if custom_hooks is not None: self.add_custom_hooks(custom_hooks) - self.conditions = {} - - if conditions is not None: self.add_conditions(conditions) - self.is_local = [] - - def add_loadables(self, loadables): - self.loadables.update(loadables) - - def add_paths(self, paths): - self.paths.update(paths) - - def add_custom_hooks(self, custom_hooks): - self.custom_hooks.update(custom_hooks) - - def add_conditions(self, conditions): - self.conditions.update(conditions) - - @staticmethod - def split_path(path): - def split(src): - if "/" in src: return src.rsplit("/", maxsplit=1) - else: return "./", src - - return split(path) - - def collect_files(self, default_source=None): - loadable_paths = {} - for name in self.loadables: - if not self.is_loadable(name): continue - save_filename = name + ".ckpt" - - if name in self.paths: - source, filename = self.split_path( - self.paths[name] - ) - elif default_source is not None: - filename = save_filename - source = default_source - else: raise ValueError - - fetch_kwargs = { - "filename": filename, - "source": source - } - path = None - - def run_fetch(**kwargs): - nonlocal path - - path = fetch(**kwargs) - - run_on_main( - run_fetch, - kwargs=fetch_kwargs, - post_func=run_fetch, - post_kwargs=fetch_kwargs - ) - - loadable_paths[name] = path - self.paths[name] = str(path) - self.is_local.append(name) - - return loadable_paths - - def is_loadable(self, name): - if name not in self.conditions: return True - condition = self.conditions[name] - - if callable(condition): return condition() - else: return bool(condition) - - def load_collected(self): - paramfiles = {} - for name in self.loadables: - if not self.is_loadable(name): continue - - if name in self.is_local: paramfiles[name] = self.paths[name] - else: raise ValueError - - self._call_load_hooks(paramfiles) - - def _call_load_hooks(self, paramfiles): - for name, obj in self.loadables.items(): - if not self.is_loadable(name): continue - loadpath = paramfiles[name] - - if name in self.custom_hooks: - self.custom_hooks[name](obj, loadpath) - continue - - default_hook = get_default_hook(obj, DEFAULT_TRANSFER_HOOKS) - - if default_hook is not None: - default_hook(obj, loadpath) - continue - - default_hook = get_default_hook(obj, DEFAULT_LOAD_HOOKS) - - if default_hook is not None: - end_of_epoch = False - default_hook(obj, loadpath, end_of_epoch) - continue - - raise RuntimeError \ No newline at end of file diff --git a/main/library/speaker_diarization/segment.py b/main/library/speaker_diarization/segment.py deleted file mode 100644 index 49dbb816fc02f453850b1b6a6eaefc65d5250ed1..0000000000000000000000000000000000000000 --- a/main/library/speaker_diarization/segment.py +++ /dev/null @@ -1,613 +0,0 @@ -import os -import sys -import numpy as np - -sys.path.append(os.getcwd()) - -from main.tools.sortedcontainers import SortedList - -class Timeline: - @classmethod - def from_df(cls, df, uri = None): - return cls(segments=list(df['segment']), uri=uri) - - def __init__( - self, - segments = None, - uri = None - ): - if segments is None: segments = () - segments_set = set([segment for segment in segments if segment]) - - self.segments_set_ = segments_set - self.segments_list_ = SortedList(segments_set) - self.segments_boundaries_ = SortedList(( - boundary - for segment in segments_set - for boundary in segment - )) - self.uri = uri - - def __len__(self): - return len(self.segments_set_) - - def __nonzero__(self): - return self.__bool__() - - def __bool__(self): - return len(self.segments_set_) > 0 - - def __iter__(self): - return iter(self.segments_list_) - - def __getitem__(self, k): - return self.segments_list_[k] - - def __eq__(self, other): - return self.segments_set_ == other.segments_set_ - - def __ne__(self, other): - return self.segments_set_ != other.segments_set_ - - def index(self, segment): - return self.segments_list_.index(segment) - - def add(self, segment): - segments_set_ = self.segments_set_ - if segment in segments_set_ or not segment: return self - - segments_set_.add(segment) - self.segments_list_.add(segment) - - segments_boundaries_ = self.segments_boundaries_ - segments_boundaries_.add(segment.start) - segments_boundaries_.add(segment.end) - - return self - - def remove(self, segment): - segments_set_ = self.segments_set_ - if segment not in segments_set_: return self - - segments_set_.remove(segment) - self.segments_list_.remove(segment) - - segments_boundaries_ = self.segments_boundaries_ - segments_boundaries_.remove(segment.start) - segments_boundaries_.remove(segment.end) - - return self - - def discard(self, segment): - return self.remove(segment) - - def __ior__(self, timeline): - return self.update(timeline) - - def update(self, timeline): - segments_set = self.segments_set_ - segments_set |= timeline.segments_set_ - - self.segments_list_ = SortedList(segments_set) - self.segments_boundaries_ = SortedList(( - boundary - for segment in segments_set - for boundary in segment - )) - - return self - - def __or__(self, timeline): - return self.union(timeline) - - def union(self, timeline): - return Timeline( - segments=self.segments_set_ | timeline.segments_set_, - uri=self.uri - ) - - def co_iter(self, other): - for segment in self.segments_list_: - temp = Segment(start=segment.end, end=segment.end) - - for other_segment in other.segments_list_.irange(maximum=temp): - if segment.intersects(other_segment): yield segment, other_segment - - def crop_iter(self, support, mode = 'intersection', returns_mapping = False): - if mode not in {'loose', 'strict', 'intersection'}: raise ValueError - if not isinstance(support, (Segment, Timeline)): raise TypeError - - if isinstance(support, Segment): - support = Timeline( - segments=([support] if support else []), - uri=self.uri - ) - - for yielded in self.crop_iter( - support, - mode=mode, - returns_mapping=returns_mapping - ): - yield yielded - - return - - support = support.support() - - if mode == 'loose': - for segment, _ in self.co_iter(support): - yield segment - - return - - if mode == 'strict': - for segment, other_segment in self.co_iter(support): - if segment in other_segment: yield segment - - return - - for segment, other_segment in self.co_iter(support): - mapped_to = segment & other_segment - if not mapped_to: continue - - if returns_mapping: yield segment, mapped_to - else: yield mapped_to - - def crop(self, support, mode = 'intersection', returns_mapping = False): - if mode == 'intersection' and returns_mapping: - segments, mapping = [], {} - - for segment, mapped_to in self.crop_iter( - support, - mode='intersection', - returns_mapping=True - ): - segments.append(mapped_to) - mapping[mapped_to] = mapping.get(mapped_to, list()) + [segment] - - return Timeline(segments=segments, uri=self.uri), mapping - - return Timeline(segments=self.crop_iter(support, mode=mode), uri=self.uri) - - def overlapping(self, t): - return list(self.overlapping_iter(t)) - - def overlapping_iter(self, t): - for segment in self.segments_list_.irange(maximum=Segment(start=t, end=t)): - if segment.overlaps(t): yield segment - - def get_overlap(self): - overlaps_tl = Timeline(uri=self.uri) - - for s1, s2 in self.co_iter(self): - if s1 == s2: continue - - overlaps_tl.add(s1 & s2) - - return overlaps_tl.support() - - def extrude(self, removed, mode = 'intersection'): - if isinstance(removed, Segment): removed = Timeline([removed]) - - if mode == "loose": mode = "strict" - elif mode == "strict": mode = "loose" - - return self.crop( - removed.gaps( - support=Timeline( - [self.extent()], - uri=self.uri - ) - ), - mode=mode - ) - - def __str__(self): - n = len(self.segments_list_) - string = "[" - - for i, segment in enumerate(self.segments_list_): - string += str(segment) - string += "\n " if i + 1 < n else "" - - string += "]" - return string - - def __repr__(self): - return "" % (self.uri, list(self.segments_list_)) - - def __contains__(self, included): - if isinstance(included, Segment): return included in self.segments_set_ - elif isinstance(included, Timeline): return self.segments_set_.issuperset(included.segments_set_) - else: raise TypeError - - def empty(self): - return Timeline(uri=self.uri) - - def covers(self, other): - gaps = self.gaps(support=other.extent()) - - for _ in gaps.co_iter(other): - return False - - return True - - def copy(self, segment_func = None): - if segment_func is None: return Timeline(segments=self.segments_list_, uri=self.uri) - return Timeline(segments=[segment_func(s) for s in self.segments_list_], uri=self.uri) - - def extent(self): - if self.segments_set_: - segments_boundaries_ = self.segments_boundaries_ - return Segment(start=segments_boundaries_[0], end=segments_boundaries_[-1]) - - return Segment(start=0.0, end=0.0) - - def support_iter(self, collar = 0.0): - if not self: return - - new_segment = self.segments_list_[0] - - for segment in self: - possible_gap = segment ^ new_segment - - if not possible_gap or possible_gap.duration < collar: new_segment |= segment - else: - yield new_segment - new_segment = segment - - yield new_segment - - def support(self, collar = 0.): - return Timeline(segments=self.support_iter(collar), uri=self.uri) - - def duration(self): - return sum(s.duration for s in self.support_iter()) - - def gaps_iter(self, support = None): - if support is None: support = self.extent() - if not isinstance(support, (Segment, Timeline)): raise TypeError - - if isinstance(support, Segment): - end = support.start - - for segment in self.crop(support, mode='intersection').support(): - gap = Segment(start=end, end=segment.start) - if gap: yield gap - - end = segment.end - - gap = Segment(start=end, end=support.end) - if gap: yield gap - elif isinstance(support, Timeline): - for segment in support.support(): - for gap in self.gaps_iter(support=segment): - yield gap - - def gaps(self, support = None): - return Timeline(segments=self.gaps_iter(support=support), uri=self.uri) - - def segmentation(self): - support = self.support() - timestamps = set([]) - - for (start, end) in self: - timestamps.add(start) - timestamps.add(end) - - timestamps = sorted(timestamps) - if len(timestamps) == 0: return Timeline(uri=self.uri) - - segments = [] - start = timestamps[0] - - for end in timestamps[1:]: - segment = Segment(start=start, end=end) - - if segment and support.overlapping(segment.middle): segments.append(segment) - start = end - - return Timeline(segments=segments, uri=self.uri) - - def _iter_uem(self): - uri = self.uri if self.uri else "" - - for segment in self: - yield f"{uri} 1 {segment.start:.3f} {segment.end:.3f}\n" - - def to_uem(self): - return "".join([line for line in self._iter_uem()]) - - def write_uem(self, file): - for line in self._iter_uem(): - file.write(line) - - def _repr_png_(self): - return None - -class Segment: - def __init__(self, start, end): - self.start = start - self.end = end - - @staticmethod - def set_precision(ndigits = None): - global AUTO_ROUND_TIME, SEGMENT_PRECISION - - if ndigits is None: - AUTO_ROUND_TIME = False - SEGMENT_PRECISION = 1e-6 - else: - AUTO_ROUND_TIME = True - SEGMENT_PRECISION = 10 ** (-ndigits) - - def __bool__(self): - return bool((self.end - self.start) > SEGMENT_PRECISION) - - def __post_init__(self): - if AUTO_ROUND_TIME: - object.__setattr__(self, 'start', int(self.start / SEGMENT_PRECISION + 0.5) * SEGMENT_PRECISION) - object.__setattr__(self, 'end', int(self.end / SEGMENT_PRECISION + 0.5) * SEGMENT_PRECISION) - - @property - def duration(self): - return self.end - self.start if self else 0. - - @property - def middle(self): - return .5 * (self.start + self.end) - - def __iter__(self): - yield self.start - yield self.end - - def copy(self): - return Segment(start=self.start, end=self.end) - - def __contains__(self, other): - return (self.start <= other.start) and (self.end >= other.end) - - def __and__(self, other): - return Segment(start=max(self.start, other.start), end=min(self.end, other.end)) - - def intersects(self, other): - return ( - self.start < other.start and other.start < self.end - SEGMENT_PRECISION - ) or ( - self.start > other.start and self.start < other.end - SEGMENT_PRECISION - ) or ( - self.start == other.start - ) - - def overlaps(self, t): - return self.start <= t and self.end >= t - - def __or__(self, other): - if not self: return other - if not other: return self - - return Segment( - start=min(self.start, other.start), - end=max(self.end, other.end) - ) - - def __xor__(self, other): - if (not self) or (not other): raise ValueError - - return Segment( - start=min(self.end, other.end), - end=max(self.start, other.start) - ) - - def _str_helper(self, seconds): - from datetime import timedelta - - negative = seconds < 0 - td = timedelta(seconds=abs(seconds)) - - hours, remainder = divmod(td.seconds + 86400 * td.days, 3600) - minutes, seconds = divmod(remainder, 60) - - return '%s%02d:%02d:%02d.%03d' % ('-' if negative else ' ', hours, minutes, seconds, td.microseconds / 1000) - - def __str__(self): - if self: return '[%s --> %s]' % (self._str_helper(self.start), self._str_helper(self.end)) - return '[]' - - def __repr__(self): - return '' % (self.start, self.end) - - def _repr_png_(self): - return None - -class SlidingWindow: - def __init__( - self, - duration=0.030, - step=0.010, - start=0.000, - end=None - ): - if duration <= 0: raise ValueError - self.__duration = duration - if step <= 0: raise ValueError - - self.__step = step - self.__start = start - - if end is None: self.__end = np.inf - else: - if end <= start: raise ValueError - self.__end = end - - self.__i = -1 - - @property - def start(self): - return self.__start - - @property - def end(self): - return self.__end - - @property - def step(self): - return self.__step - - @property - def duration(self): - return self.__duration - - def closest_frame(self, t): - return int(np.rint((t - self.__start - .5 * self.__duration) / self.__step)) - - def samples(self, from_duration, mode = 'strict'): - if mode == 'strict': return int(np.floor((from_duration - self.duration) / self.step)) + 1 - elif mode == 'loose': return int(np.floor((from_duration + self.duration) / self.step)) - elif mode == 'center': return int(np.rint((from_duration / self.step))) - - def crop(self, focus, mode = 'loose', fixed = None, return_ranges = False): - if not isinstance(focus, (Segment, Timeline)): raise TypeError - - if isinstance(focus, Timeline): - if fixed is not None: raise ValueError - - if return_ranges: - ranges = [] - - for i, s in enumerate(focus.support()): - rng = self.crop(s, mode=mode, fixed=fixed, return_ranges=True) - - if i == 0 or rng[0][0] > ranges[-1][1]: ranges += rng - else: ranges[-1][1] = rng[0][1] - - return ranges - - return np.unique( - np.hstack([ - self.crop( - s, - mode=mode, - fixed=fixed, - return_ranges=False - ) - for s in focus.support() - ]) - ) - - if mode == 'loose': - i = int(np.ceil((focus.start - self.duration - self.start) / self.step)) - - if fixed is None: - j = int(np.floor((focus.end - self.start) / self.step)) - rng = (i, j + 1) - else: - n = self.samples(fixed, mode='loose') - rng = (i, i + n) - elif mode == 'strict': - i = int(np.ceil((focus.start - self.start) / self.step)) - - if fixed is None: - j = int(np.floor((focus.end - self.duration - self.start) / self.step)) - rng = (i, j + 1) - else: - n = self.samples(fixed, mode='strict') - rng = (i, i + n) - elif mode == 'center': - i = self.closest_frame(focus.start) - - if fixed is None: - j = self.closest_frame(focus.end) - rng = (i, j + 1) - else: - n = self.samples(fixed, mode='center') - rng = (i, i + n) - else: raise ValueError - - if return_ranges: return [list(rng)] - return np.array(range(*rng), dtype=np.int64) - - def segmentToRange(self, segment): - return self.segment_to_range(segment) - - def segment_to_range(self, segment): - return self.closest_frame(segment.start), int(segment.duration / self.step) + 1 - - def rangeToSegment(self, i0, n): - return self.range_to_segment(i0, n) - - def range_to_segment(self, i0, n): - start = self.__start + (i0 - .5) * self.__step + .5 * self.__duration - - if i0 == 0: start = self.start - return Segment(start, start + (n * self.__step)) - - def samplesToDuration(self, nSamples): - return self.samples_to_duration(nSamples) - - def samples_to_duration(self, n_samples): - return self.range_to_segment(0, n_samples).duration - - def durationToSamples(self, duration): - return self.duration_to_samples(duration) - - def duration_to_samples(self, duration): - return self.segment_to_range(Segment(0, duration))[1] - - def __getitem__(self, i): - start = self.__start + i * self.__step - if start >= self.__end: return None - - return Segment(start=start, end=start + self.__duration) - - def next(self): - return self.__next__() - - def __next__(self): - self.__i += 1 - window = self[self.__i] - - if window: return window - else: raise StopIteration() - - def __iter__(self): - self.__i = -1 - return self - - def __len__(self): - if np.isinf(self.__end): raise ValueError - i = self.closest_frame(self.__end) - - while (self[i]): - i += 1 - - length = i - return length - - def copy(self): - return self.__class__( - duration=self.duration, - step=self.step, - start=self.start, - end=self.end - ) - - def __call__(self, support, align_last = False): - if isinstance(support, Timeline): segments = support - elif isinstance(support, Segment): segments = Timeline(segments=[support]) - else: raise TypeError - - for segment in segments: - if segment.duration < self.duration: continue - - for s in SlidingWindow( - duration=self.duration, - step=self.step, - start=segment.start, - end=segment.end - ): - if s in segment: - yield s - last = s - - if align_last and last.end < segment.end: yield Segment(start=segment.end - self.duration, end=segment.end) \ No newline at end of file diff --git a/main/library/speaker_diarization/speechbrain.py b/main/library/speaker_diarization/speechbrain.py deleted file mode 100644 index d27be79c1049725e48f26d04afe65d7bcef2824e..0000000000000000000000000000000000000000 --- a/main/library/speaker_diarization/speechbrain.py +++ /dev/null @@ -1,327 +0,0 @@ -import os -import torch -import torchaudio - -from functools import wraps -from types import SimpleNamespace -from torch.nn import SyncBatchNorm -from hyperpyyaml import load_hyperpyyaml - -from torch.nn import DataParallel as DP -from torch.nn.parallel import DistributedDataParallel as DDP - -MAIN_PROC_ONLY = 0 - -def fetch(filename, source): - return os.path.abspath(os.path.join(source, filename)) - -def run_on_main( - func, - args=None, - kwargs=None, - post_func=None, - post_args=None, - post_kwargs=None, - run_post_on_main=False -): - if args is None: args = [] - if kwargs is None: kwargs = {} - if post_args is None: post_args = [] - if post_kwargs is None: post_kwargs = {} - - main_process_only(func)(*args, **kwargs) - ddp_barrier() - - if post_func is not None: - if run_post_on_main: post_func(*post_args, **post_kwargs) - else: - if not if_main_process(): post_func(*post_args, **post_kwargs) - ddp_barrier() - -def is_distributed_initialized(): - return ( - torch.distributed.is_available() and - torch.distributed.is_initialized() - ) - -def if_main_process(): - if is_distributed_initialized(): return torch.distributed.get_rank() == 0 - else: return True - -class MainProcessContext: - def __enter__(self): - global MAIN_PROC_ONLY - - MAIN_PROC_ONLY += 1 - return self - - def __exit__(self, exc_type, exc_value, traceback): - global MAIN_PROC_ONLY - - MAIN_PROC_ONLY -= 1 - -def main_process_only(function): - @wraps(function) - def main_proc_wrapped_func(*args, **kwargs): - with MainProcessContext(): - return function(*args, **kwargs) if if_main_process() else None - - return main_proc_wrapped_func - -def ddp_barrier(): - if MAIN_PROC_ONLY >= 1 or not is_distributed_initialized(): return - - if torch.distributed.get_backend() == torch.distributed.Backend.NCCL: - torch.distributed.barrier( - device_ids=[torch.cuda.current_device()] - ) - elif hasattr(torch, "xpu") and hasattr(torch.distributed.Backend, "XCCL") and torch.distributed.get_backend() == torch.distributed.Backend.XCCL: - torch.distributed.barrier( - device_ids=[torch.xpu.current_device()] - ) - else: torch.distributed.barrier() - -class Resample(torch.nn.Module): - def __init__( - self, - orig_freq=16000, - new_freq=16000, - *args, - **kwargs - ): - super().__init__() - - self.orig_freq = orig_freq - self.new_freq = new_freq - self.resampler = torchaudio.transforms.Resample( - orig_freq=orig_freq, - new_freq=new_freq, - *args, - **kwargs - ) - - def forward(self, waveforms): - if self.orig_freq == self.new_freq: return waveforms - - unsqueezed = False - if len(waveforms.shape) == 2: - waveforms = waveforms.unsqueeze(1) - unsqueezed = True - elif len(waveforms.shape) == 3: waveforms = waveforms.transpose(1, 2) - else: raise ValueError - - self.resampler.to(waveforms.device) - resampled_waveform = self.resampler(waveforms) - - return resampled_waveform.squeeze(1) if unsqueezed else resampled_waveform.transpose(1, 2) - -class AudioNormalizer: - def __init__( - self, - sample_rate=16000, - mix="avg-to-mono" - ): - self.sample_rate = sample_rate - - if mix not in ["avg-to-mono", "keep"]: raise ValueError - - self.mix = mix - self._cached_resamplers = {} - - def __call__(self, audio, sample_rate): - if sample_rate not in self._cached_resamplers: - self._cached_resamplers[ - sample_rate - ] = Resample( - sample_rate, - self.sample_rate - ) - - return self._mix( - self._cached_resamplers[sample_rate](audio.unsqueeze(0)).squeeze(0) - ) - - def _mix(self, audio): - flat_input = audio.dim() == 1 - - if self.mix == "avg-to-mono": - if flat_input: return audio - return torch.mean(audio, 1) - - if self.mix == "keep": return audio - -class Pretrained(torch.nn.Module): - HPARAMS_NEEDED, MODULES_NEEDED = [], [] - - def __init__( - self, - modules=None, - hparams=None, - run_opts=None, - freeze_params=True - ): - super().__init__() - - for arg, default in { - "device": "cpu", - "data_parallel_count": -1, - "data_parallel_backend": False, - "distributed_launch": False, - "distributed_backend": "nccl", - "jit": False, - "jit_module_keys": None, - "compile": False, - "compile_module_keys": None, - "compile_mode": "reduce-overhead", - "compile_using_fullgraph": False, - "compile_using_dynamic_shape_tracing": False - }.items(): - if run_opts is not None and arg in run_opts: setattr(self, arg, run_opts[arg]) - elif hparams is not None and arg in hparams: setattr(self, arg, hparams[arg]) - else: setattr(self, arg, default) - - self.mods = torch.nn.ModuleDict(modules) - - for module in self.mods.values(): - if module is not None: module.to(self.device) - - if self.HPARAMS_NEEDED and hparams is None: raise ValueError - - if hparams is not None: - for hp in self.HPARAMS_NEEDED: - if hp not in hparams: raise ValueError - - self.hparams = SimpleNamespace(**hparams) - - self._prepare_modules(freeze_params) - self.audio_normalizer = hparams.get("audio_normalizer", AudioNormalizer()) - - def _prepare_modules(self, freeze_params): - self._compile() - self._wrap_distributed() - - if freeze_params: - self.mods.eval() - for p in self.mods.parameters(): - p.requires_grad = False - - def _compile(self): - compile_available = hasattr(torch, "compile") - if not compile_available and self.compile_module_keys is not None: raise ValueError - - compile_module_keys = set() - if self.compile: compile_module_keys = set(self.mods) if self.compile_module_keys is None else set(self.compile_module_keys) - - jit_module_keys = set() - if self.jit: jit_module_keys = set(self.mods) if self.jit_module_keys is None else set(self.jit_module_keys) - - for name in compile_module_keys | jit_module_keys: - if name not in self.mods: raise ValueError - - for name in compile_module_keys: - try: - module = torch.compile( - self.mods[name], - mode=self.compile_mode, - fullgraph=self.compile_using_fullgraph, - dynamic=self.compile_using_dynamic_shape_tracing - ) - except Exception: - continue - - self.mods[name] = module.to(self.device) - jit_module_keys.discard(name) - - for name in jit_module_keys: - module = torch.jit.script(self.mods[name]) - self.mods[name] = module.to(self.device) - - def _compile_jit(self): - self._compile() - - def _wrap_distributed(self): - if not self.distributed_launch and not self.data_parallel_backend: return - elif self.distributed_launch: - for name, module in self.mods.items(): - if any(p.requires_grad for p in module.parameters()): - self.mods[name] = DDP( - SyncBatchNorm.convert_sync_batchnorm(module), - device_ids=[self.device] - ) - else: - for name, module in self.mods.items(): - if any(p.requires_grad for p in module.parameters()): - self.mods[name] = ( - DP(module) - ) if self.data_parallel_count == -1 else ( - DP( - module, - [i for i in range(self.data_parallel_count)] - ) - ) - - @classmethod - def from_hparams( - cls, - source, - hparams_file="hyperparams.yaml", - overrides={}, - download_only=False, - overrides_must_match=True, - **kwargs - ): - with open(fetch(filename=hparams_file, source=source)) as fin: - hparams = load_hyperpyyaml( - fin, - overrides, - overrides_must_match=overrides_must_match - ) - - pretrainer = hparams.get("pretrainer", None) - - if pretrainer is not None: - run_on_main(pretrainer.collect_files, kwargs={"default_source": source}) - if not download_only: - pretrainer.load_collected() - return cls(hparams["modules"], hparams, **kwargs) - else: return cls(hparams["modules"], hparams, **kwargs) - -class EncoderClassifier(Pretrained): - MODULES_NEEDED = [ - "compute_features", - "mean_var_norm", - "embedding_model", - "classifier" - ] - - def encode_batch(self, wavs, wav_lens=None, normalize=False): - if len(wavs.shape) == 1: wavs = wavs.unsqueeze(0) - if wav_lens is None: wav_lens = torch.ones(wavs.shape[0], device=self.device) - - wavs, wav_lens = wavs.to(self.device), wav_lens.to(self.device) - wavs = wavs.float() - - embeddings = self.mods.embedding_model( - self.mods.mean_var_norm( - self.mods.compute_features(wavs), - wav_lens - ), - wav_lens - ) - - if normalize: - embeddings = self.hparams.mean_var_norm_emb( - embeddings, - torch.ones(embeddings.shape[0], device=self.device) - ) - - return embeddings - - def classify_batch(self, wavs, wav_lens=None): - out_prob = self.mods.classifier(self.encode_batch(wavs, wav_lens)).squeeze(1) - score, index = out_prob.max(dim=-1) - - return out_prob, score, index, self.hparams.label_encoder.decode_torch(index) - - def forward(self, wavs, wav_lens=None): - return self.classify_batch(wavs, wav_lens) \ No newline at end of file diff --git a/main/library/speaker_diarization/whisper.py b/main/library/speaker_diarization/whisper.py deleted file mode 100644 index 8445ed893dbe005671fa33f2dd48d5df3a8ca93d..0000000000000000000000000000000000000000 --- a/main/library/speaker_diarization/whisper.py +++ /dev/null @@ -1,1993 +0,0 @@ -import os -import sys -import gzip -import zlib -import tqdm -import torch -import base64 -import string -import tiktoken -import itertools - -import numba as nb -import numpy as np -import torch.nn as nn -import torch.nn.functional as F - -from dataclasses import replace -from torch.distributions import Categorical -from functools import cached_property, lru_cache - -sys.path.append(os.getcwd()) - -from main.app.variables import config, configs, logger - -LANGUAGES = { - "en": "english", - "zh": "chinese", - "de": "german", - "es": "spanish", - "ru": "russian", - "ko": "korean", - "fr": "french", - "ja": "japanese", - "pt": "portuguese", - "tr": "turkish", - "pl": "polish", - "ca": "catalan", - "nl": "dutch", - "ar": "arabic", - "sv": "swedish", - "it": "italian", - "id": "indonesian", - "hi": "hindi", - "fi": "finnish", - "vi": "vietnamese", - "he": "hebrew", - "uk": "ukrainian", - "el": "greek", - "ms": "malay", - "cs": "czech", - "ro": "romanian", - "da": "danish", - "hu": "hungarian", - "ta": "tamil", - "no": "norwegian", - "th": "thai", - "ur": "urdu", - "hr": "croatian", - "bg": "bulgarian", - "lt": "lithuanian", - "la": "latin", - "mi": "maori", - "ml": "malayalam", - "cy": "welsh", - "sk": "slovak", - "te": "telugu", - "fa": "persian", - "lv": "latvian", - "bn": "bengali", - "sr": "serbian", - "az": "azerbaijani", - "sl": "slovenian", - "kn": "kannada", - "et": "estonian", - "mk": "macedonian", - "br": "breton", - "eu": "basque", - "is": "icelandic", - "hy": "armenian", - "ne": "nepali", - "mn": "mongolian", - "bs": "bosnian", - "kk": "kazakh", - "sq": "albanian", - "sw": "swahili", - "gl": "galician", - "mr": "marathi", - "pa": "punjabi", - "si": "sinhala", - "km": "khmer", - "sn": "shona", - "yo": "yoruba", - "so": "somali", - "af": "afrikaans", - "oc": "occitan", - "ka": "georgian", - "be": "belarusian", - "tg": "tajik", - "sd": "sindhi", - "gu": "gujarati", - "am": "amharic", - "yi": "yiddish", - "lo": "lao", - "uz": "uzbek", - "fo": "faroese", - "ht": "haitian creole", - "ps": "pashto", - "tk": "turkmen", - "nn": "nynorsk", - "mt": "maltese", - "sa": "sanskrit", - "lb": "luxembourgish", - "my": "myanmar", - "bo": "tibetan", - "tl": "tagalog", - "mg": "malagasy", - "as": "assamese", - "tt": "tatar", - "haw": "hawaiian", - "ln": "lingala", - "ha": "hausa", - "ba": "bashkir", - "jw": "javanese", - "su": "sundanese", - "yue": "cantonese" -} - -TO_LANGUAGE_CODE = { - **{ - language: code for code, language in LANGUAGES.items() - }, - "burmese": "my", - "valencian": "ca", - "flemish": "nl", - "haitian": "ht", - "letzeburgesch": "lb", - "pushto": "ps", - "panjabi": "pa", - "moldavian": "ro", - "moldovan": "ro", - "sinhalese": "si", - "castilian": "es", - "mandarin": "zh" -} - -_ALIGNMENT_HEADS = { - "tiny.en": b"ABzY8J1N>@0{>%R00Bk>$p{7v037`oCl~+#00", - "tiny": b"ABzY8bu8Lr0{>%RKn9Fp%m@SkK7Kt=7ytkO", - "base.en": b"ABzY8;40c<0{>%RzzG;p*o+Vo09|#PsxSZm00", - "base": b"ABzY8KQ!870{>%RzyTQH3`Q^yNP!>##QT-?_)10{>%RpeA61k&I|OI3I$65C{;;pbCHh0B{qLQ;+}v00", - "small": b"ABzY8DmU6=0{>%Rpa?J`kvJ6qF(V^F86#Xh7JUGMK}P%R7%R7}kK1fFL7w6%<-Pf*t^=N)Qr&0RR9", - "large-v1": b"ABzY8r9j$a0{>%R7#4sLmoOs{s)o3~84-RPdcFk!JR%R7=D0pU<_bnWW*tkYAhobTNnu$jnkEkXqp)j;w1Tzk)UH3X%SZd&fFZ2fC2yj", - "large-v3": b"ABzY8gWO1E0{>%R7(9S+Kn!D~%ngiGaR?*L!iJG9p-nab0JQ=-{D1-g00", - "large": b"ABzY8gWO1E0{>%R7(9S+Kn!D~%ngiGaR?*L!iJG9p-nab0JQ=-{D1-g00", - "large-v3-turbo": b"ABzY8j^C+e0{>%RARaKHP%t(lGR*)0g!tONPyhe`" -} - -SAMPLE_RATE, N_FFT, HOP_LENGTH, CHUNK_LENGTH = 16000, 400, 160, 30 -N_SAMPLES = CHUNK_LENGTH * SAMPLE_RATE -N_SAMPLES_PER_TOKEN = HOP_LENGTH * 2 -stft = None - -def exact_div(x, y): - assert x % y == 0 - return x // y - -N_FRAMES = exact_div(N_SAMPLES, HOP_LENGTH) -FRAMES_PER_SECOND = exact_div(SAMPLE_RATE, HOP_LENGTH) -TOKENS_PER_SECOND = exact_div(SAMPLE_RATE, N_SAMPLES_PER_TOKEN) - -def load_model(name = "base", device = "cpu"): - checkpoint_file = os.path.join(configs["speaker_diarization_path"], "models", name + ".pt") - alignment_heads = _ALIGNMENT_HEADS[name] - - with open(checkpoint_file, "rb") as fp: - checkpoint = torch.load(fp, map_location="cpu", weights_only=True) - - del checkpoint_file - - model = Whisper(ModelDimensions(**checkpoint["dims"])) - model.load_state_dict(checkpoint["model_state_dict"]) - model.set_alignment_heads(alignment_heads) - - return model.to(device) - -def merge_punctuations(alignment, prepended, appended): - i = len(alignment) - 2 - j = len(alignment) - 1 - - while i >= 0: - previous = alignment[i] - following = alignment[j] - - if previous.word.startswith(" ") and previous.word.strip() in prepended: - following.word = previous.word + following.word - following.tokens = previous.tokens + following.tokens - - previous.word = "" - previous.tokens = [] - else: j = i - - i -= 1 - - i = 0 - j = 1 - - while j < len(alignment): - previous = alignment[i] - following = alignment[j] - - if not previous.word.endswith(" ") and following.word in appended: - previous.word = previous.word + following.word - previous.tokens = previous.tokens + following.tokens - - following.word = "" - following.tokens = [] - else: i = j - - j += 1 - -class WordTiming: - def __init__( - self, - word, - tokens, - start, - end, - probability - ): - self.word = word - self.tokens = tokens - self.start = start - self.end = end - self.probability = probability - -def median_filter(x, filter_width): - pad_width = filter_width // 2 - - if x.shape[-1] <= pad_width: return x - if (ndim := x.ndim) <= 2: x = x[None, None, :] - - assert (filter_width > 0 and filter_width % 2 == 1) - result = None - - x = F.pad( - x, - (filter_width // 2, filter_width // 2, 0, 0), - mode="reflect" - ) - - if result is None: result = x.unfold(-1, filter_width, 1).sort()[0][..., filter_width // 2] - if ndim <= 2: result = result[0, 0] - - return result - -@nb.jit(nopython=True) -def backtrace(trace): - i = trace.shape[0] - 1 - j = trace.shape[1] - 1 - - trace[0, :] = 2 - trace[:, 0] = 1 - - result = [] - while i > 0 or j > 0: - result.append((i - 1, j - 1)) - - if trace[i, j] == 0: - i -= 1 - j -= 1 - elif trace[i, j] == 1: i -= 1 - elif trace[i, j] == 2: j -= 1 - else: raise ValueError - - return np.array(result)[::-1, :].T - - -@nb.jit(nopython=True, parallel=True) -def dtw_cpu(x): - N, M = x.shape - - cost = np.ones((N + 1, M + 1), dtype=np.float32) * np.inf - trace = -np.ones((N + 1, M + 1), dtype=np.float32) - cost[0, 0] = 0 - - for j in range(1, M + 1): - for i in range(1, N + 1): - c0 = cost[i - 1, j - 1] - c1 = cost[i - 1, j] - c2 = cost[i, j - 1] - - if c0 < c1 and c0 < c2: c, t = c0, 0 - elif c1 < c0 and c1 < c2: c, t = c1, 1 - else: c, t = c2, 2 - - cost[i, j] = x[i - 1, j - 1] + c - trace[i, j] = t - - return backtrace(trace) - -def dtw(x): - return dtw_cpu(x.double().cpu().numpy()) - -def find_alignment(model, tokenizer, text_tokens, mel, num_frames, *, medfilt_width = 7, qk_scale = 1.0): - if len(text_tokens) == 0: return [] - - tokens = torch.tensor([ - *tokenizer.sot_sequence, - tokenizer.no_timestamps, - *text_tokens, - tokenizer.eot - ]).to(model.device) - - QKs = [None] * model.dims.n_text_layer - hooks = [ - block.cross_attn.register_forward_hook( - lambda _, ins, outs, index=i: QKs.__setitem__(index, outs[-1][0]) - ) - for i, block in enumerate(model.decoder.blocks) - ] - - with torch.no_grad(): - token_probs = model( - mel.unsqueeze(0), - tokens.unsqueeze(0) - )[0][len(tokenizer.sot_sequence) :, : tokenizer.eot].softmax(dim=-1) - - text_token_probs = token_probs[np.arange(len(text_tokens)), text_tokens].tolist() - - for hook in hooks: - hook.remove() - - if not config.device.startswith(("privateuseone", "ocl")): - alignment_indices = model.alignment_heads.indices().T - else: - alignment_indices = [ - (l, h) - for l in range(model.alignment_heads.size(0)) - for h in range(model.alignment_heads.size(1)) - if model.alignment_heads[l, h] - ] - - weights = ( - torch.stack([ - QKs[_l][_h] - for _l, _h in alignment_indices - ])[:, :, : num_frames // 2] * qk_scale - ).softmax(dim=-1) - - std, mean = torch.std_mean(weights, dim=-2, keepdim=True, unbiased=False) - - if config.device.startswith("privateuseone"): - weights = median_filter( - ((weights - mean) / std).cpu(), - medfilt_width - ).to(weights.device) - else: - weights = median_filter( - (weights - mean) / std, - medfilt_width - ) - - text_indices, time_indices = dtw(-weights.mean(axis=0)[len(tokenizer.sot_sequence) : -1]) - words, word_tokens = tokenizer.split_to_word_tokens(text_tokens + [tokenizer.eot]) - if len(word_tokens) <= 1: return [] - - word_boundaries = np.pad(np.cumsum([len(t) for t in word_tokens[:-1]]), (1, 0)) - jump_times = time_indices[np.pad(np.diff(text_indices), (1, 0), constant_values=1).astype(bool)] / TOKENS_PER_SECOND - - return [ - WordTiming( - word, - tokens, - start, - end, - probability - ) - for word, tokens, start, end, probability in zip( - words, - word_tokens, - jump_times[word_boundaries[:-1]], - jump_times[word_boundaries[1:]], - [ - np.mean(text_token_probs[i:j]) - for i, j in zip( - word_boundaries[:-1], - word_boundaries[1:] - ) - ] - ) - ] - -def add_word_timestamps( - *, - segments, - model, - tokenizer, - mel, - num_frames, - prepend_punctuations = "\"'“¿([{-", - append_punctuations = "\"'.。,,!!??::”)]}、", - last_speech_timestamp, - **kwargs -): - if len(segments) == 0: return - - text_tokens_per_segment = [ - [token for token in segment["tokens"] if token < tokenizer.eot] - for segment in segments - ] - - text_tokens = list(itertools.chain.from_iterable(text_tokens_per_segment)) - alignment = find_alignment(model, tokenizer, text_tokens, mel, num_frames, **kwargs) - - word_durations = np.array([t.end - t.start for t in alignment]) - word_durations = word_durations[word_durations.nonzero()] - - median_duration = min(0.7, float(np.median(word_durations) if len(word_durations) > 0 else 0.0)) - max_duration = median_duration * 2 - - if len(word_durations) > 0: - sentence_end_marks = ".。!!??" - for i in range(1, len(alignment)): - if alignment[i].end - alignment[i].start > max_duration: - if alignment[i].word in sentence_end_marks: alignment[i].end = alignment[i].start + max_duration - elif alignment[i - 1].word in sentence_end_marks: alignment[i].start = alignment[i].end - max_duration - - merge_punctuations(alignment, prepend_punctuations, append_punctuations) - time_offset = segments[0]["seek"] * HOP_LENGTH / SAMPLE_RATE - word_index = 0 - - for segment, text_tokens in zip(segments, text_tokens_per_segment): - saved_tokens = 0 - words = [] - - while word_index < len(alignment) and saved_tokens < len(text_tokens): - timing = alignment[word_index] - - if timing.word: - words.append( - dict( - word=timing.word, - start=round(time_offset + timing.start, 2), - end=round(time_offset + timing.end, 2), - probability=timing.probability - ) - ) - - saved_tokens += len(timing.tokens) - word_index += 1 - - if len(words) > 0: - if ( - words[0]["end"] - last_speech_timestamp > median_duration * 4 and ( - words[0]["end"] - words[0]["start"] > max_duration or - (len(words) > 1 and words[1]["end"] - words[0]["start"] > max_duration * 2) - ) - ): - if ( - len(words) > 1 and - words[1]["end"] - words[1]["start"] > max_duration - ): - words[0]["end"] = words[1]["start"] = max(words[1]["end"] / 2, words[1]["end"] - max_duration) - - words[0]["start"] = max(0, words[0]["end"] - max_duration) - - if ( - segment["start"] < words[0]["end"] and - segment["start"] - 0.5 > words[0]["start"] - ): - words[0]["start"] = max(0, min(words[0]["end"] - median_duration, segment["start"])) - else: - segment["start"] = words[0]["start"] - - if ( - segment["end"] > words[-1]["start"] and - segment["end"] + 0.5 < words[-1]["end"] - ): - words[-1]["end"] = max(words[-1]["start"] + median_duration, segment["end"]) - else: - segment["end"] = words[-1]["end"] - - last_speech_timestamp = segment["end"] - - segment["words"] = words - -@lru_cache(maxsize=None) -def mel_filters(device, n_mels): - assert n_mels in {80, 128} - - with np.load( - os.path.join(configs["speaker_diarization_path"], "assets", "mel_filters.npz"), - allow_pickle=False - ) as f: - return torch.from_numpy(f[f"mel_{n_mels}"]).to(device) - -def log_mel_spectrogram(audio, n_mels = 80, padding = 0, device = None): - global stft - - if not torch.is_tensor(audio): - if isinstance(audio, str): - from main.library.audio.audio import load_audio - audio = load_audio(audio, sample_rate=SAMPLE_RATE) - - audio = torch.from_numpy(audio).float() - - if device is not None: audio = audio.to(device) - if padding > 0: audio = F.pad(audio, (0, padding)) - - if str(audio.device).startswith(("ocl", "privateuseone")): - if stft is None: - from main.library.backends.utils import STFT - - stft = STFT( - N_FFT, - HOP_LENGTH, - N_FFT - ).to(audio.device) - - fft = stft.transform( - audio.unsqueeze(0), - eps=1e-9 - ).squeeze(0) - else: - fft = torch.stft( - audio, - n_fft=N_FFT, - hop_length=HOP_LENGTH, - window=torch.hann_window(N_FFT).to(audio.device), - return_complex=True - ) - - log_spec = ( - mel_filters(audio.device, n_mels) @ fft[..., :-1].abs() ** 2 - ).clamp(min=1e-10).log10() - - return (log_spec.maximum(log_spec.max() - 8.0) + 4.0) / 4.0 - -def pad_or_trim(array, length = N_SAMPLES, *, axis = -1): - if torch.is_tensor(array): - if array.shape[axis] > length: - array = array.index_select( - dim=axis, - index=torch.arange(length, device=array.device) - ) - - if array.shape[axis] < length: - pad_widths = [(0, 0)] * array.ndim - pad_widths[axis] = (0, length - array.shape[axis]) - array = F.pad(array, [pad for sizes in pad_widths[::-1] for pad in sizes]) - else: - if array.shape[axis] > length: array = array.take(indices=range(length), axis=axis) - - if array.shape[axis] < length: - pad_widths = [(0, 0)] * array.ndim - pad_widths[axis] = (0, length - array.shape[axis]) - array = np.pad(array, pad_widths) - - return array - -def get_end(segments): - return next(( - w["end"] - for s in reversed(segments) - for w in reversed(s["words"]) - ), segments[-1]["end"] if segments else None) - -def transcribe_function( - model, - audio, - *, - verbose = None, - temperature = (0.0, 0.2, 0.4, 0.6, 0.8, 1.0), - compression_ratio_threshold = 2.4, - logprob_threshold = -1.0, - no_speech_threshold = 0.6, - condition_on_previous_text = True, - initial_prompt = None, - carry_initial_prompt = False, - word_timestamps = False, - prepend_punctuations = "\"'“¿([{-", - append_punctuations = "\"'.。,,!!??::”)]}、", - clip_timestamps = "0", - hallucination_silence_threshold = None, - fp16 = False, - **decode_options -): - dtype = torch.float16 if fp16 else torch.float32 - decode_options["fp16"] = fp16 - - mel = log_mel_spectrogram(audio, model.dims.n_mels, padding=N_SAMPLES) - content_frames = mel.shape[-1] - N_FRAMES - content_duration = float(content_frames * HOP_LENGTH / SAMPLE_RATE) - - if decode_options.get("language", None) is None: - if not model.is_multilingual: decode_options["language"] = "vi" - else: - mel_segment = pad_or_trim(mel, N_FRAMES).to(model.device).to(dtype) - _, probs = model.detect_language(mel_segment) - decode_options["language"] = max(probs, key=probs.get) - - if verbose is not None: logger.info(f"{LANGUAGES[decode_options['language']].title()}") - - language = decode_options["language"] - task = decode_options.get("task", "transcribe") - - tokenizer = get_tokenizer( - model.is_multilingual, - num_languages=model.num_languages, - language=language, - task=task - ) - - if isinstance(clip_timestamps, str): - clip_timestamps = [ - float(ts) - for ts in ( - clip_timestamps.split(",") if clip_timestamps else [] - ) - ] - - seek_points = [round(ts * FRAMES_PER_SECOND) for ts in clip_timestamps] - - if len(seek_points) == 0: seek_points.append(0) - if len(seek_points) % 2 == 1: seek_points.append(content_frames) - - seek_clips = list(zip(seek_points[::2], seek_points[1::2])) - punctuation = "\"'“¿([{-\"'.。,,!!??::”)]}、" - - def decode_with_fallback(segment): - decode_result = None - temperatures = ( - [temperature] if isinstance(temperature, (int, float)) else temperature - ) - - for t in temperatures: - kwargs = {**decode_options} - - if t > 0: - kwargs.pop("beam_size", None) - kwargs.pop("patience", None) - else: kwargs.pop("best_of", None) - - decode_result = model.decode( - segment, - DecodingOptions( - **kwargs, - temperature=t - ) - ) - - needs_fallback = False - - if ( - compression_ratio_threshold is not None and - decode_result.compression_ratio > compression_ratio_threshold - ): - needs_fallback = True - - if ( - logprob_threshold is not None and - decode_result.avg_logprob < logprob_threshold - ): - needs_fallback = True - - if ( - no_speech_threshold is not None and - decode_result.no_speech_prob > no_speech_threshold and - logprob_threshold is not None and - decode_result.avg_logprob < logprob_threshold - ): - needs_fallback = False - - if not needs_fallback: break - - return decode_result - - clip_idx = 0 - seek = seek_clips[clip_idx][0] - - input_stride = exact_div(N_FRAMES, model.dims.n_audio_ctx) - time_precision = (input_stride * HOP_LENGTH / SAMPLE_RATE) - - all_tokens, all_segments = [], [] - prompt_reset_since = 0 - - remaining_prompt_length = model.dims.n_text_ctx // 2 - 1 - - if initial_prompt is not None: - initial_prompt_tokens = tokenizer.encode(" " + initial_prompt.strip()) - all_tokens.extend(initial_prompt_tokens) - remaining_prompt_length -= len(initial_prompt_tokens) - else: initial_prompt_tokens = [] - - def new_segment(*, start, end, tokens, result): - tokens = tokens.tolist() - - return { - "seek": seek, - "start": start, - "end": end, - "text": tokenizer.decode([token for token in tokens if token < tokenizer.eot]), - "tokens": tokens, - "temperature": result.temperature, - "avg_logprob": result.avg_logprob, - "compression_ratio": result.compression_ratio, - "no_speech_prob": result.no_speech_prob - } - - with tqdm.tqdm(total=content_frames, unit="frames", disable=verbose is not False) as pbar: - last_speech_timestamp = 0.0 - while clip_idx < len(seek_clips): - seek_clip_start, seek_clip_end = seek_clips[clip_idx] - if seek < seek_clip_start: seek = seek_clip_start - - if seek >= seek_clip_end: - clip_idx += 1 - if clip_idx < len(seek_clips): seek = seek_clips[clip_idx][0] - continue - - time_offset = float(seek * HOP_LENGTH / SAMPLE_RATE) - window_end_time = float((seek + N_FRAMES) * HOP_LENGTH / SAMPLE_RATE) - - segment_size = min(N_FRAMES, content_frames - seek, seek_clip_end - seek) - mel_segment = mel[:, seek : seek + segment_size] - - segment_duration = segment_size * HOP_LENGTH / SAMPLE_RATE - mel_segment = pad_or_trim(mel_segment, N_FRAMES).to(model.device).to(dtype) - - if carry_initial_prompt: - decode_options["prompt"] = initial_prompt_tokens + all_tokens[max(len(initial_prompt_tokens), prompt_reset_since):][-remaining_prompt_length:] - else: - decode_options["prompt"] = all_tokens[prompt_reset_since:] - - result = decode_with_fallback(mel_segment) - tokens = torch.tensor(result.tokens) - - if no_speech_threshold is not None: - should_skip = result.no_speech_prob > no_speech_threshold - if (logprob_threshold is not None and result.avg_logprob > logprob_threshold): - should_skip = False - - if should_skip: - seek += segment_size - continue - - previous_seek = seek - current_segments = [] - - def word_anomaly_score(word): - probability = word.get("probability", 0.0) - duration = word["end"] - word["start"] - score = 0.0 - - if probability < 0.15: score += 1.0 - if duration < 0.133: score += (0.133 - duration) * 15 - if duration > 2.0: score += duration - 2.0 - - return score - - def is_segment_anomaly(segment): - if segment is None or not segment["words"]: return False - - words = [w for w in segment["words"] if w["word"] not in punctuation] - words = words[:8] - - score = sum(word_anomaly_score(w) for w in words) - - return score >= 3 or score + 0.01 >= len(words) - - def next_words_segment(segments): - return next((s for s in segments if s["words"]), None) - - timestamp_tokens = tokens.ge(tokenizer.timestamp_begin) - single_timestamp_ending = timestamp_tokens[-2:].tolist() == [False, True] - - consecutive = torch.where(timestamp_tokens[:-1] & timestamp_tokens[1:])[0] - consecutive.add_(1) - - if len(consecutive) > 0: - slices = consecutive.tolist() - if single_timestamp_ending: - slices.append(len(tokens)) - - last_slice = 0 - for current_slice in slices: - sliced_tokens = tokens[last_slice:current_slice] - current_segments.append( - new_segment( - start=time_offset + (sliced_tokens[0].item() - tokenizer.timestamp_begin) * time_precision, - end=time_offset + (sliced_tokens[-1].item() - tokenizer.timestamp_begin) * time_precision, - tokens=sliced_tokens, - result=result - ) - ) - last_slice = current_slice - - if single_timestamp_ending: seek += segment_size - else: seek += (tokens[last_slice - 1].item() - tokenizer.timestamp_begin) * input_stride - else: - duration = segment_duration - - timestamps = tokens[timestamp_tokens.nonzero().flatten()] - if (len(timestamps) > 0 and timestamps[-1].item() != tokenizer.timestamp_begin): - duration = (timestamps[-1].item() - tokenizer.timestamp_begin) * time_precision - - current_segments.append( - new_segment( - start=time_offset, - end=time_offset + duration, - tokens=tokens, - result=result - ) - ) - - seek += segment_size - - if word_timestamps: - add_word_timestamps( - segments=current_segments, - model=model, - tokenizer=tokenizer, - mel=mel_segment, - num_frames=segment_size, - prepend_punctuations=prepend_punctuations, - append_punctuations=append_punctuations, - last_speech_timestamp=last_speech_timestamp - ) - - if not single_timestamp_ending: - last_word_end = get_end(current_segments) - if last_word_end is not None and last_word_end > time_offset: - seek = round(last_word_end * FRAMES_PER_SECOND) - - if hallucination_silence_threshold is not None: - threshold = hallucination_silence_threshold - - if not single_timestamp_ending: - last_word_end = get_end(current_segments) - if last_word_end is not None and last_word_end > time_offset: - seek = round(last_word_end * FRAMES_PER_SECOND) if (window_end_time - last_word_end) > threshold else (previous_seek + segment_size) - - first_segment = next_words_segment(current_segments) - if first_segment is not None and is_segment_anomaly(first_segment): - gap = first_segment["start"] - time_offset - - if gap > threshold: - seek = previous_seek + round(gap * FRAMES_PER_SECOND) - continue - - hal_last_end = last_speech_timestamp - - for si in range(len(current_segments)): - segment = current_segments[si] - if not segment["words"]: continue - - if is_segment_anomaly(segment): - next_segment = next_words_segment(current_segments[si + 1 :]) - hal_next_start = next_segment["words"][0]["start"] if next_segment is not None else (time_offset + segment_duration) - - if ( - segment["start"] - hal_last_end > threshold or segment["start"] < threshold or segment["start"] - time_offset < 2.0 - ) and ( - hal_next_start - segment["end"] > threshold or is_segment_anomaly(next_segment) or window_end_time - segment["end"] < 2.0 - ): - seek = round(max(time_offset + 1, segment["start"]) * FRAMES_PER_SECOND) - if content_duration - segment["end"] < threshold: seek = content_frames - - current_segments[si:] = [] - break - - hal_last_end = segment["end"] - - last_word_end = get_end(current_segments) - if last_word_end is not None: last_speech_timestamp = last_word_end - - for _, segment in enumerate(current_segments): - if segment["start"] == segment["end"] or segment["text"].strip() == "": - segment["text"] = "" - segment["tokens"] = [] - segment["words"] = [] - - all_segments.extend([{"id": i, **segment} for i, segment in enumerate(current_segments, start=len(all_segments))]) - all_tokens.extend([token for segment in current_segments for token in segment["tokens"]]) - - if not condition_on_previous_text or result.temperature > 0.5: prompt_reset_since = len(all_tokens) - pbar.update(min(content_frames, seek) - previous_seek) - - return dict(text=tokenizer.decode(all_tokens[len(initial_prompt_tokens) :]), segments=all_segments, language=language) - -def compression_ratio(text): - text_bytes = text.encode("utf-8") - return len(text_bytes) / len(zlib.compress(text_bytes)) - -def sinusoids(length, channels, max_timescale=10000): - assert channels % 2 == 0 - - scaled_time = torch.arange(length)[:, np.newaxis] * (-(np.log(max_timescale) / (channels // 2 - 1)) * torch.arange(channels // 2)).exp()[np.newaxis, :] - return torch.cat([scaled_time.sin(), scaled_time.cos()], dim=1) - -@torch.no_grad() -def detect_language_function(model, mel, tokenizer = None): - if tokenizer is None: tokenizer = get_tokenizer(model.is_multilingual, num_languages=model.num_languages) - if (tokenizer.language is None or tokenizer.language_token not in tokenizer.sot_sequence): raise ValueError - - single = mel.ndim == 2 - - if single: mel = mel.unsqueeze(0) - if mel.shape[-2:] != (model.dims.n_audio_ctx, model.dims.n_audio_state): mel = model.encoder(mel) - - n_audio = mel.shape[0] - logits = model.logits(torch.tensor([[tokenizer.sot]] * n_audio).to(mel.device), mel)[:, 0] - - mask = torch.ones(logits.shape[-1], dtype=torch.bool) - mask[list(tokenizer.all_language_tokens)] = False - - logits[:, mask] = -np.inf - - language_tokens = logits.argmax(dim=-1) - language_probs = [ - { - c: logits.softmax(dim=-1).cpu()[i, j].item() - for j, c in zip( - tokenizer.all_language_tokens, - tokenizer.all_language_codes - ) - } - for i in range(n_audio) - ] - - if single: - language_tokens = language_tokens[0] - language_probs = language_probs[0] - - return language_tokens, language_probs - -@lru_cache(maxsize=None) -def get_tokenizer(multilingual, *, num_languages = 99, language = None, task = None): - if language is not None: - language = language.lower() - if language not in LANGUAGES: - if language in TO_LANGUAGE_CODE: language = TO_LANGUAGE_CODE[language] - else: raise ValueError - - if multilingual: - encoding_name = "multilingual" - language = language or "en" - task = task or "transcribe" - else: - encoding_name = "gpt2" - language = None - task = None - - return Tokenizer( - encoding_name=encoding_name, - num_languages=num_languages, - language=language, - task=task - ) - -@lru_cache(maxsize=None) -def get_encoding(name = "gpt2", num_languages = 99): - vocab_path = os.path.join(configs["speaker_diarization_path"], "assets", f"{name}.tiktoken") - - ranks = { - base64.b64decode(token): int(rank) - for token, rank in ( - line.split() - for line in open(vocab_path) - if line - ) - } - - n_vocab = len(ranks) - special_tokens = {} - - specials = [ - "<|endoftext|>", "<|startoftranscript|>", - *[f"<|{lang}|>" for lang in list(LANGUAGES.keys())[:num_languages]], - "<|translate|>", - "<|transcribe|>", - "<|startoflm|>", - "<|startofprev|>", - "<|nospeech|>", - "<|notimestamps|>", - *[f"<|{i * 0.02:.2f}|>" for i in range(1501)] - ] - - for token in specials: - special_tokens[token] = n_vocab - n_vocab += 1 - - return tiktoken.Encoding( - name=os.path.basename(vocab_path), - explicit_n_vocab=n_vocab, - pat_str=r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""", - mergeable_ranks=ranks, - special_tokens=special_tokens - ) - -class DecodingOptions: - def __init__( - self, - task = "transcribe", - language = None, - temperature = 0.0, - sample_len = None, - best_of = None, - beam_size = None, - patience = None, - length_penalty = None, - prompt = None, - prefix = None, - suppress_tokens = "-1", - suppress_blank = True, - without_timestamps = False, - max_initial_timestamp = 1.0, - fp16 = False - ): - self.task = task - self.language = language - self.temperature = temperature - self.sample_len = sample_len - self.best_of = best_of - self.beam_size = beam_size - self.patience = patience - self.length_penalty = length_penalty - self.prompt = prompt - self.prefix = prefix - self.suppress_tokens = suppress_tokens - self.suppress_blank = suppress_blank - self.without_timestamps = without_timestamps - self.max_initial_timestamp = max_initial_timestamp - self.fp16 = fp16 - -@torch.no_grad() -def decode_function(model, mel, options = DecodingOptions(), **kwargs): - if single := mel.ndim == 2: mel = mel.unsqueeze(0) - if kwargs: options = replace(options, **kwargs) - - result = DecodingTask(model, options).run(mel) - return result[0] if single else result - -class ModelDimensions: - def __init__( - self, - n_mels, - n_audio_ctx, - n_audio_state, - n_audio_head, - n_audio_layer, - n_vocab, - n_text_ctx, - n_text_state, - n_text_head, - n_text_layer - ): - self.n_mels = n_mels - self.n_audio_ctx = n_audio_ctx - self.n_audio_state = n_audio_state - self.n_audio_head = n_audio_head - self.n_audio_layer = n_audio_layer - self.n_vocab = n_vocab - self.n_text_ctx = n_text_ctx - self.n_text_state = n_text_state - self.n_text_head = n_text_head - self.n_text_layer = n_text_layer - -class LayerNorm(nn.LayerNorm): - def forward( - self, - x - ): - return super().forward( - x.float() - ).type(x.dtype) - -class Linear(nn.Linear): - def forward( - self, - x - ): - return F.linear( - x, - self.weight.to(x.dtype), - self.bias.to(x.dtype) if self.bias is not None else None - ) - -class Conv1d(nn.Conv1d): - def _conv_forward( - self, - x, - weight, - bias - ): - return super()._conv_forward( - x, - weight.to(x.dtype), - bias.to(x.dtype) if bias is not None else None - ) - -class TextDecoder(nn.Module): - def __init__( - self, - n_vocab, - n_ctx, - n_state, - n_head, - n_layer - ): - super().__init__() - self.token_embedding = nn.Embedding(n_vocab, n_state) - self.positional_embedding = nn.Parameter(torch.empty(n_ctx, n_state)) - self.blocks = nn.ModuleList([ - ResidualAttentionBlock( - n_state, - n_head, - cross_attention=True - ) - for _ in range(n_layer) - ]) - self.ln = LayerNorm(n_state) - self.register_buffer("mask", torch.empty(n_ctx, n_ctx).fill_(-np.inf).triu_(1), persistent=False) - - def forward(self, x, xa, kv_cache = None): - offset = next(iter(kv_cache.values())).shape[1] if kv_cache else 0 - x = (self.token_embedding(x) + self.positional_embedding[offset : offset + x.shape[-1]]).to(xa.dtype) - - for block in self.blocks: - x = block(x, xa, mask=self.mask, kv_cache=kv_cache) - - x = self.ln(x) - return (x @ self.token_embedding.weight.to(x.dtype).transpose(0, 1)).float() - -class AudioEncoder(nn.Module): - def __init__(self, n_mels, n_ctx, n_state, n_head, n_layer): - super().__init__() - self.conv1 = Conv1d(n_mels, n_state, kernel_size=3, padding=1) - self.conv2 = Conv1d(n_state, n_state, kernel_size=3, stride=2, padding=1) - self.register_buffer("positional_embedding", sinusoids(n_ctx, n_state)) - self.blocks = nn.ModuleList([ - ResidualAttentionBlock( - n_state, - n_head - ) - for _ in range(n_layer) - ]) - self.ln_post = LayerNorm(n_state) - - def forward(self, x): - x = F.gelu(self.conv2(F.gelu(self.conv1(x)))).permute(0, 2, 1) - - assert x.shape[1:] == self.positional_embedding.shape - x = (x + self.positional_embedding).to(x.dtype) - - for block in self.blocks: - x = block(x) - - return self.ln_post(x) - -class Whisper(nn.Module): - def __init__(self, dims): - super().__init__() - self.dims = dims - self.encoder = AudioEncoder( - self.dims.n_mels, - self.dims.n_audio_ctx, - self.dims.n_audio_state, - self.dims.n_audio_head, - self.dims.n_audio_layer - ) - self.decoder = TextDecoder( - self.dims.n_vocab, - self.dims.n_text_ctx, - self.dims.n_text_state, - self.dims.n_text_head, - self.dims.n_text_layer - ) - - all_heads = torch.zeros(self.dims.n_text_layer, self.dims.n_text_head, dtype=torch.bool) - all_heads[self.dims.n_text_layer // 2 :] = True - self.register_buffer("alignment_heads", all_heads if config.device.startswith(("privateuseone", "ocl")) else all_heads.to_sparse(), persistent=False) - - def set_alignment_heads(self, dump): - alignment = torch.from_numpy( - np.frombuffer( - gzip.decompress(base64.b85decode(dump)), dtype=bool - ).copy() - ).reshape(self.dims.n_text_layer, self.dims.n_text_head) - - if not config.device.startswith(("privateuseone", "ocl")): alignment = alignment.to_sparse() - self.register_buffer("alignment_heads", alignment, persistent=False) - - def embed_audio(self, mel): - return self.encoder(mel) - - def logits(self, tokens, audio_features): - return self.decoder(tokens, audio_features) - - def forward(self, mel, tokens): - return self.decoder(tokens, self.encoder(mel)) - - @property - def device(self): - return next(self.parameters()).device - - @property - def is_multilingual(self): - return self.dims.n_vocab >= 51865 - - @property - def num_languages(self): - return self.dims.n_vocab - 51765 - int(self.is_multilingual) - - def install_kv_cache_hooks(self, cache = None): - cache = {**cache} if cache is not None else {} - hooks = [] - - def save_to_cache(module, _, output): - cache[module] = output if module not in cache or output.shape[1] > self.dims.n_text_ctx else torch.cat([cache[module], output], dim=1).detach() - return cache[module] - - def install_hooks(layer: nn.Module): - if isinstance(layer, MultiHeadAttention): - hooks.append(layer.key.register_forward_hook(save_to_cache)) - hooks.append(layer.value.register_forward_hook(save_to_cache)) - - self.decoder.apply(install_hooks) - return cache, hooks - - detect_language = detect_language_function - transcribe = transcribe_function - decode = decode_function - -class ResidualAttentionBlock(nn.Module): - def __init__( - self, - n_state, - n_head, - cross_attention = False - ): - super().__init__() - self.attn = MultiHeadAttention(n_state, n_head) - self.attn_ln = LayerNorm(n_state) - self.cross_attn = (MultiHeadAttention(n_state, n_head) if cross_attention else None) - self.cross_attn_ln = LayerNorm(n_state) if cross_attention else None - - n_mlp = n_state * 4 - self.mlp = nn.Sequential(Linear(n_state, n_mlp), nn.GELU(), Linear(n_mlp, n_state)) - self.mlp_ln = LayerNorm(n_state) - - def forward(self, x, xa = None, mask = None, kv_cache = None): - x = x + self.attn(self.attn_ln(x), mask=mask, kv_cache=kv_cache)[0] - if self.cross_attn: x = x + self.cross_attn(self.cross_attn_ln(x), xa, kv_cache=kv_cache)[0] - - return x + self.mlp(self.mlp_ln(x)) - -class MultiHeadAttention(nn.Module): - def __init__( - self, - n_state, - n_head - ): - super().__init__() - self.n_head = n_head - self.query = Linear(n_state, n_state) - self.key = Linear(n_state, n_state, bias=False) - self.value = Linear(n_state, n_state) - self.out = Linear(n_state, n_state) - - def forward( - self, - x, - xa = None, - mask = None, - kv_cache = None - ): - k, v = ( - self.key(x if xa is None else xa), - self.value(x if xa is None else xa) - ) if kv_cache is None or xa is None or self.key not in kv_cache else ( - kv_cache[self.key], - kv_cache[self.value] - ) - - wv, qk = self.qkv_attention(self.query(x), k, v, mask) - return self.out(wv), qk - - def qkv_attention(self, q, k, v, mask = None): - _, n_ctx, n_state = q.shape - scale = (n_state // self.n_head) ** -0.25 - - q, k, v = ( - q.view(*q.shape[:2], self.n_head, -1).permute(0, 2, 1, 3), - k.view(*k.shape[:2], self.n_head, -1).permute(0, 2, 1, 3), - v.view(*v.shape[:2], self.n_head, -1).permute(0, 2, 1, 3) - ) - - qk = (q * scale) @ (k * scale).transpose(-1, -2) - if mask is not None: qk = qk + mask[:n_ctx, :n_ctx] - qk = qk.float() - - return (F.softmax(qk, dim=-1).to(q.dtype) @ v).permute(0, 2, 1, 3).flatten(start_dim=2), qk.detach() - -class LogitFilter: - def apply(self, logits, tokens): - pass - -class SuppressBlank(LogitFilter): - def __init__(self, tokenizer, sample_begin): - self.tokenizer = tokenizer - self.sample_begin = sample_begin - - def apply(self, logits, tokens): - if tokens.shape[1] == self.sample_begin: - logits[:, self.tokenizer.encode(" ") + [self.tokenizer.eot]] = -np.inf - -class SuppressTokens(LogitFilter): - def __init__(self, suppress_tokens): - self.suppress_tokens = list(suppress_tokens) - - def apply(self, logits, tokens): - logits[:, self.suppress_tokens] = -np.inf - -class Inference: - def logits(self, tokens, audio_features): - pass - - def rearrange_kv_cache(self, source_indices): - pass - - def cleanup_caching(self): - pass - -class PyTorchInference(Inference): - def __init__(self, model, initial_token_length): - self.model = model - self.initial_token_length = initial_token_length - self.kv_cache = {} - self.hooks = [] - - self.kv_modules = [ - block.attn.key - for block in self.model.decoder.blocks - ] + [ - block.attn.value - for block in self.model.decoder.blocks - ] - - def logits(self, tokens, audio_features): - if not self.kv_cache: self.kv_cache, self.hooks = self.model.install_kv_cache_hooks() - if tokens.shape[-1] > self.initial_token_length: tokens = tokens[:, -1:] - - return self.model.decoder(tokens, audio_features, kv_cache=self.kv_cache) - - def cleanup_caching(self): - for hook in self.hooks: - hook.remove() - - self.kv_cache = {} - self.hooks = [] - - def rearrange_kv_cache(self, source_indices): - if source_indices != list(range(len(source_indices))): - for module in self.kv_modules: - self.kv_cache[module] = self.kv_cache[module][source_indices].detach() - -class SequenceRanker: - def rank(self, tokens, sum_logprobs): - pass - -class MaximumLikelihoodRanker(SequenceRanker): - def __init__( - self, - length_penalty - ): - self.length_penalty = length_penalty - - def rank(self, tokens, sum_logprobs): - def scores(logprobs, lengths): - result = [] - - for logprob, length in zip(logprobs, lengths): - result.append( - logprob / (length if self.length_penalty is None else ((5 + length) / 6) ** self.length_penalty) - ) - - return result - - return [ - np.argmax(scores(p, l)) - for p, l in zip(sum_logprobs, [[len(t) for t in s] for s in tokens]) - ] - -class TokenDecoder: - def reset(self): - pass - - def update(self, tokens, logits, sum_logprobs): - pass - - def finalize(self, tokens, sum_logprobs): - pass - -class GreedyDecoder(TokenDecoder): - def __init__( - self, - temperature, - eot - ): - self.temperature = temperature - self.eot = eot - - def update(self, tokens, logits, sum_logprobs): - next_tokens = logits.argmax(dim=-1) if self.temperature == 0 else ( - Categorical( - logits=(logits / self.temperature).cpu() if config.device.startswith("ocl") else (logits / self.temperature) - ) - ).sample().to(logits.device) - - logprobs = F.log_softmax(logits.float(), dim=-1) - sum_logprobs += logprobs[torch.arange(logprobs.shape[0]), next_tokens] * (tokens[:, -1] != self.eot) - - next_tokens[tokens[:, -1] == self.eot] = self.eot - tokens = torch.cat([tokens, next_tokens[:, None]], dim=-1) - - return tokens, (tokens[:, -1] == self.eot).all() - - def finalize(self, tokens, sum_logprobs): - return F.pad(tokens, (0, 1), value=self.eot), sum_logprobs.tolist() - -class BeamSearchDecoder(TokenDecoder): - def __init__( - self, - beam_size, - eot, - inference, - patience = None - ): - self.beam_size = beam_size - self.eot = eot - self.inference = inference - self.patience = patience or 1.0 - self.max_candidates = round(beam_size * self.patience) - self.finished_sequences = None - - assert (self.max_candidates > 0) - - def reset(self): - self.finished_sequences = None - - def update(self, tokens, logits, sum_logprobs): - if tokens.shape[0] % self.beam_size != 0: raise ValueError(f"{tokens.shape}[0] % {self.beam_size} != 0") - - n_audio = tokens.shape[0] // self.beam_size - if self.finished_sequences is None: self.finished_sequences = [{} for _ in range(n_audio)] - - logprobs = F.log_softmax(logits.float(), dim=-1) - next_tokens, source_indices, finished_sequences = [], [], [] - - for i in range(n_audio): - scores, sources, finished = {}, {}, {} - - for j in range(self.beam_size): - idx = i * self.beam_size + j - prefix = tokens[idx].tolist() - for logprob, token in zip(*logprobs[idx].topk(self.beam_size + 1)): - sequence = tuple(prefix + [token.item()]) - scores[sequence] = (sum_logprobs[idx] + logprob).item() - sources[sequence] = idx - - saved = 0 - - for sequence in sorted(scores, key=scores.get, reverse=True): - if sequence[-1] == self.eot: finished[sequence] = scores[sequence] - else: - sum_logprobs[len(next_tokens)] = scores[sequence] - next_tokens.append(sequence) - source_indices.append(sources[sequence]) - - saved += 1 - if saved == self.beam_size: break - - finished_sequences.append(finished) - - self.inference.rearrange_kv_cache(source_indices) - assert len(self.finished_sequences) == len(finished_sequences) - - for previously_finished, newly_finished in zip(self.finished_sequences, finished_sequences): - for seq in sorted(newly_finished, key=newly_finished.get, reverse=True): - if len(previously_finished) >= self.max_candidates: break - previously_finished[seq] = newly_finished[seq] - - return ( - torch.tensor(next_tokens, device=tokens.device), - all(len(sequences) >= self.max_candidates for sequences in self.finished_sequences) - ) - - def finalize(self, preceding_tokens, sum_logprobs): - sum_logprobs = sum_logprobs.cpu() - - for i, sequences in enumerate(self.finished_sequences): - if (len(sequences) < self.beam_size): - for j in list(np.argsort(sum_logprobs[i]))[::-1]: - sequence = preceding_tokens[i, j].tolist() + [self.eot] - sequences[tuple(sequence)] = sum_logprobs[i][j].item() - if len(sequences) >= self.beam_size: break - - return ( - [[torch.tensor(seq) for seq in sequences.keys()] for sequences in self.finished_sequences], - [list(sequences.values()) for sequences in self.finished_sequences] - ) - -class ApplyTimestampRules(LogitFilter): - def __init__( - self, - tokenizer, - sample_begin, - max_initial_timestamp_index - ): - self.tokenizer = tokenizer - self.sample_begin = sample_begin - self.max_initial_timestamp_index = max_initial_timestamp_index - - def apply(self, logits, tokens): - if self.tokenizer.no_timestamps is not None: logits[:, self.tokenizer.no_timestamps] = -np.inf - - for k in range(tokens.shape[0]): - sampled_tokens = tokens[k, self.sample_begin :] - seq = [t for t in sampled_tokens.tolist()] - - last_was_timestamp = (len(seq) >= 1 and seq[-1] >= self.tokenizer.timestamp_begin) - penultimate_was_timestamp = (len(seq) < 2 or seq[-2] >= self.tokenizer.timestamp_begin) - - if last_was_timestamp: - if penultimate_was_timestamp: logits[k, self.tokenizer.timestamp_begin :] = -np.inf - else: logits[k, : self.tokenizer.eot] = -np.inf - - timestamps = sampled_tokens[sampled_tokens.ge(self.tokenizer.timestamp_begin)] - - if timestamps.numel() > 0: - logits[ - k, - self.tokenizer.timestamp_begin : - timestamps[-1] if last_was_timestamp and not penultimate_was_timestamp else (timestamps[-1] + 1) - ] = -np.inf - - if tokens.shape[1] == self.sample_begin: - logits[:, : self.tokenizer.timestamp_begin] = -np.inf - - if self.max_initial_timestamp_index is not None: - last_allowed = (self.tokenizer.timestamp_begin + self.max_initial_timestamp_index) - logits[:, last_allowed + 1 :] = -np.inf - - logprobs = F.log_softmax(logits.float(), dim=-1) - for k in range(tokens.shape[0]): - if logprobs[k, self.tokenizer.timestamp_begin :].logsumexp(dim=-1) > logprobs[k, : self.tokenizer.timestamp_begin].max(): - logits[k, : self.tokenizer.timestamp_begin] = -np.inf - -class DecodingTask: - def __init__( - self, - model, - options - ): - self.model = model - language = options.language or "en" - - tokenizer = get_tokenizer( - model.is_multilingual, - num_languages=model.num_languages, - language=language, - task=options.task - ) - - self.tokenizer = tokenizer - self.options = self._verify_options(options) - self.n_group = options.beam_size or options.best_of or 1 - self.n_ctx = model.dims.n_text_ctx - self.sample_len = options.sample_len or model.dims.n_text_ctx // 2 - self.sot_sequence = tokenizer.sot_sequence - - if self.options.without_timestamps: - self.sot_sequence = tokenizer.sot_sequence_including_notimestamps - - self.initial_tokens = self._get_initial_tokens() - self.sample_begin = len(self.initial_tokens) - self.sot_index = self.initial_tokens.index(tokenizer.sot) - - self.inference = PyTorchInference(model, len(self.initial_tokens)) - self.sequence_ranker = MaximumLikelihoodRanker(options.length_penalty) - - self.decoder = BeamSearchDecoder( - options.beam_size, - tokenizer.eot, - self.inference, - options.patience - ) if options.beam_size is not None else GreedyDecoder( - options.temperature, - tokenizer.eot - ) - - self.logit_filters = [] - - if self.options.suppress_blank: - self.logit_filters.append( - SuppressBlank( - self.tokenizer, - self.sample_begin - ) - ) - - if self.options.suppress_tokens: - self.logit_filters.append( - SuppressTokens( - self._get_suppress_tokens() - ) - ) - - if not options.without_timestamps: - max_initial_timestamp_index = None - - if options.max_initial_timestamp: - max_initial_timestamp_index = round( - self.options.max_initial_timestamp / (CHUNK_LENGTH / model.dims.n_audio_ctx) - ) - - self.logit_filters.append( - ApplyTimestampRules( - tokenizer, - self.sample_begin, - max_initial_timestamp_index - ) - ) - - def _verify_options(self, options): - if options.beam_size is not None and options.best_of is not None: raise ValueError - if options.temperature == 0 and options.best_of is not None: raise ValueError - if options.patience is not None and options.beam_size is None: raise ValueError - if options.length_penalty is not None and not (0 <= options.length_penalty <= 1): raise ValueError - - return options - - def _get_initial_tokens(self): - tokens = list(self.sot_sequence) - - if prefix := self.options.prefix: - prefix_tokens = (self.tokenizer.encode(" " + prefix.strip()) if isinstance(prefix, str) else prefix) - if self.sample_len is not None: prefix_tokens = prefix_tokens[-(self.n_ctx // 2 - self.sample_len):] - tokens = tokens + prefix_tokens - - if prompt := self.options.prompt: - tokens = ( - [self.tokenizer.sot_prev] + - (self.tokenizer.encode(" " + prompt.strip()) if isinstance(prompt, str) else prompt)[-(self.n_ctx // 2 - 1) :] + tokens - ) - - return tuple(tokens) - - def _get_suppress_tokens(self): - suppress_tokens = self.options.suppress_tokens - if isinstance(suppress_tokens, str): suppress_tokens = [int(t) for t in suppress_tokens.split(",")] - - if -1 in suppress_tokens: - suppress_tokens = [t for t in suppress_tokens if t >= 0] - suppress_tokens.extend(self.tokenizer.non_speech_tokens) - elif suppress_tokens is None or len(suppress_tokens) == 0: suppress_tokens = [] - else: assert isinstance(suppress_tokens, list) - - suppress_tokens.extend([ - self.tokenizer.transcribe, - self.tokenizer.translate, - self.tokenizer.sot, - self.tokenizer.sot_prev, - self.tokenizer.sot_lm - ]) - - if self.tokenizer.no_speech is not None: suppress_tokens.append(self.tokenizer.no_speech) - return tuple(sorted(set(suppress_tokens))) - - def _get_audio_features(self, mel): - if self.options.fp16: mel = mel.half() - - audio_features = mel if mel.shape[-2:] == (self.model.dims.n_audio_ctx, self.model.dims.n_audio_state) else self.model.encoder(mel) - if audio_features.dtype != (torch.float16 if self.options.fp16 else torch.float32): return TypeError - - return audio_features - - def _detect_language(self, audio_features, tokens): - languages = [self.options.language] * audio_features.shape[0] - lang_probs = None - - if self.options.language is None or self.options.task == "lang_id": - lang_tokens, lang_probs = self.model.detect_language(audio_features, self.tokenizer) - languages = [max(probs, key=probs.get) for probs in lang_probs] - - if self.options.language is None: tokens[:, self.sot_index + 1] = lang_tokens - - return languages, lang_probs - - def _main_loop(self, audio_features, tokens): - n_batch = tokens.shape[0] - sum_logprobs = torch.zeros(n_batch, device=audio_features.device) - no_speech_probs = [np.nan] * n_batch - - try: - for i in range(self.sample_len): - logits = self.inference.logits(tokens, audio_features) - - if (i == 0 and self.tokenizer.no_speech is not None): - probs_at_sot = logits[:, self.sot_index].float().softmax(dim=-1) - no_speech_probs = probs_at_sot[:, self.tokenizer.no_speech].tolist() - - logits = logits[:, -1] - for logit_filter in self.logit_filters: - logit_filter.apply(logits.cpu() if config.device.startswith("ocl") else logits, tokens) - - tokens, completed = self.decoder.update(tokens, logits, sum_logprobs) - if completed or tokens.shape[-1] > self.n_ctx: break - finally: - self.inference.cleanup_caching() - - return tokens, sum_logprobs, no_speech_probs - - @torch.no_grad() - def run(self, mel): - self.decoder.reset() - tokenizer = self.tokenizer - n_audio = mel.shape[0] - - audio_features = self._get_audio_features(mel) - tokens = torch.tensor([self.initial_tokens]).repeat(n_audio, 1) - - languages, language_probs = self._detect_language(audio_features, tokens) - if self.options.task == "lang_id": - return [ - DecodingResult( - audio_features=features, - language=language, - language_probs=probs - ) - for features, language, probs in zip( - audio_features, - languages, - language_probs - ) - ] - - tokens = tokens.repeat_interleave(self.n_group, dim=0).to(audio_features.device) - tokens, sum_logprobs, no_speech_probs = self._main_loop(audio_features, tokens) - - audio_features = audio_features[:: self.n_group] - no_speech_probs = no_speech_probs[:: self.n_group] - - assert audio_features.shape[0] == len(no_speech_probs) == n_audio - - tokens = tokens.reshape(n_audio, self.n_group, -1) - sum_logprobs = sum_logprobs.reshape(n_audio, self.n_group) - - tokens, sum_logprobs = self.decoder.finalize(tokens, sum_logprobs) - tokens = [[t[self.sample_begin : (t == tokenizer.eot).nonzero()[0, 0]] for t in s] for s in tokens] - - selected = self.sequence_ranker.rank(tokens, sum_logprobs) - tokens = [t[i].tolist() for i, t in zip(selected, tokens)] - - fields = ( - [tokenizer.decode(t).strip() for t in tokens], - languages, - tokens, - audio_features, - [lp / (len(t) + 1) for t, lp in zip(tokens, [lp[i] for i, lp in zip(selected, sum_logprobs)])], - no_speech_probs - ) - if len(set(map(len, fields))) != 1: raise RuntimeError - - return [ - DecodingResult( - audio_features=features, - language=language, - tokens=tokens, - text=text, - avg_logprob=avg_logprob, - no_speech_prob=no_speech_prob, - temperature=self.options.temperature, - compression_ratio=compression_ratio(text) - ) - for text, language, tokens, features, avg_logprob, no_speech_prob in zip(*fields) - ] - -class DecodingResult: - def __init__( - self, - audio_features, - language, - language_probs = None, - tokens = None, - text = "", - avg_logprob = np.nan, - no_speech_prob = np.nan, - temperature = np.nan, - compression_ratio = np.nan - ): - self.audio_features = audio_features - self.language = language - self.language_probs = language_probs if language_probs is not None else {} - self.tokens = tokens if tokens is not None else [] - self.text = text - self.avg_logprob = avg_logprob - self.no_speech_prob = no_speech_prob - self.temperature = temperature - self.compression_ratio = compression_ratio - -class Tokenizer: - def __init__( - self, - encoding_name, - num_languages = 2, - language = None, - task = None, - sot_sequence = () - ): - self.encoding = get_encoding( - name=encoding_name, - num_languages=num_languages - ) - - self.num_languages = num_languages - self.language = language - self.task = task - self.sot_sequence = sot_sequence - self.special_tokens = {} - - for special in self.encoding.special_tokens_set: - special_token = self.encoding.encode_single_token(special) - self.special_tokens[special] = special_token - - sot = self.special_tokens["<|startoftranscript|>"] - langs = tuple(LANGUAGES.keys())[: self.num_languages] - sot_sequence = [sot] - - if self.language is not None: - sot_sequence.append( - sot + 1 + langs.index(self.language) - ) - - if self.task is not None: - sot_sequence.append( - self.special_tokens[ - "<|transcribe|>" - ] if self.task == "transcribe" else self.special_tokens[ - "<|translate|>" - ] - ) - - self.sot_sequence = tuple(sot_sequence) - - def encode(self, text, **kwargs): - return self.encoding.encode(text, **kwargs) - - def decode(self, token_ids, **kwargs): - return self.encoding.decode([ - t - for t in token_ids - if t < self.timestamp_begin - ], **kwargs) - - def decode_with_timestamps(self, token_ids, **kwargs): - return self.encoding.decode(token_ids, **kwargs) - - @cached_property - def eot(self): - return self.encoding.eot_token - - @cached_property - def transcribe(self): - return self.special_tokens["<|transcribe|>"] - - @cached_property - def translate(self): - return self.special_tokens["<|translate|>"] - - @cached_property - def sot(self): - return self.special_tokens["<|startoftranscript|>"] - - @cached_property - def sot_lm(self): - return self.special_tokens["<|startoflm|>"] - - @cached_property - def sot_prev(self): - return self.special_tokens["<|startofprev|>"] - - @cached_property - def no_speech(self): - return self.special_tokens["<|nospeech|>"] - - @cached_property - def no_timestamps(self): - return self.special_tokens["<|notimestamps|>"] - - @cached_property - def timestamp_begin(self): - return self.special_tokens["<|0.00|>"] - - @cached_property - def language_token(self): - if self.language is None: raise ValueError - return self.to_language_token(self.language) - - def to_language_token(self, language): - if token := self.special_tokens.get(f"<|{language}|>", None): return token - raise KeyError - - @cached_property - def all_language_tokens(self): - result = [] - for token, token_id in self.special_tokens.items(): - if token.strip("<|>") in LANGUAGES: result.append(token_id) - - return tuple(result)[: self.num_languages] - - @cached_property - def all_language_codes(self): - return tuple(self.decode([_l]).strip("<|>") for _l in self.all_language_tokens) - - @cached_property - def sot_sequence_including_notimestamps(self): - return tuple(list(self.sot_sequence) + [self.no_timestamps]) - - @cached_property - def non_speech_tokens(self): - symbols = list('"#()*+/:;<=>@[\\]^_`{|}~「」『』') - symbols += ("<< >> <<< >>> -- --- -( -[ (' (\" (( )) ((( ))) [[ ]] {{ }} ♪♪ ♪♪♪".split()) - - miscellaneous = set("♩♪♫♬♭♮♯") - assert all(0x2640 <= ord(c) <= 0x267F for c in miscellaneous) - - result = {self.encoding.encode(" -")[0], self.encoding.encode(" '")[0]} - for symbol in symbols + list(miscellaneous): - for tokens in [self.encoding.encode(symbol), self.encoding.encode(" " + symbol)]: - if len(tokens) == 1 or symbol in miscellaneous: result.add(tokens[0]) - - return tuple(sorted(result)) - - def split_to_word_tokens(self, tokens): - if self.language in {"zh", "ja", "th", "lo", "my", "yue"}: return self.split_tokens_on_unicode(tokens) - return self.split_tokens_on_spaces(tokens) - - def split_tokens_on_unicode(self, tokens): - replacement_char = "\ufffd" - - words, word_tokens, current_tokens = [], [], [] - unicode_offset = 0 - - for token in tokens: - current_tokens.append(token) - decoded = self.decode_with_timestamps(current_tokens) - - if ( - replacement_char not in decoded or - self.decode_with_timestamps(tokens)[unicode_offset + decoded.index(replacement_char)] == replacement_char - ): - words.append(decoded) - word_tokens.append(current_tokens) - current_tokens = [] - unicode_offset += len(decoded) - - return words, word_tokens - - def split_tokens_on_spaces(self, tokens): - subwords, subword_tokens_list = self.split_tokens_on_unicode(tokens) - words, word_tokens = [], [] - - for subword, subword_tokens in zip(subwords, subword_tokens_list): - if ( - subword_tokens[0] >= self.eot - ) or ( - subword.startswith(" ") - ) or ( - subword.strip() in string.punctuation - ) or len(words) == 0: - words.append(subword) - word_tokens.append(subword_tokens) - else: - words[-1] = words[-1] + subword - word_tokens[-1].extend(subword_tokens) - - return words, word_tokens \ No newline at end of file diff --git a/main/library/utils.py b/main/library/utils.py deleted file mode 100644 index d5253b5d88072bbf7f9e754c3ca4911fa59aa8e2..0000000000000000000000000000000000000000 --- a/main/library/utils.py +++ /dev/null @@ -1,365 +0,0 @@ -import os -import re -import gc -import sys -import torch -import codecs -import logging -import warnings - -sys.path.append(os.getcwd()) - -from main.tools import huggingface -from main.library.backends import directml, opencl -from main.library.algorithm.faisssearch import IndexWrapper -from main.app.variables import translations, configs, config, logger, embedders_model, whisper_model - -if not config.debug_mode: - warnings.filterwarnings("ignore") - for l in ["httpx", "httpcore"]: - logging.getLogger(l).setLevel(logging.ERROR) - -def check_assets(f0_method, hubert, predictor_onnx=False, embedders_mode="fairseq"): - predictors_url = codecs.decode( - "uggcf://uhttvatsnpr.pb/NauC/Ivrganzrfr-EIP-Cebwrpg/erfbyir/znva/cerqvpgbef/", - "rot13" - ) - embedders_url = codecs.decode( - "uggcf://uhttvatsnpr.pb/NauC/Ivrganzrfr-EIP-Cebwrpg/erfbyir/znva/rzorqqref/", - "rot13" - ) - - def download_predictor(predictor): - model_path = os.path.join(configs["predictors_path"], predictor) - - if not os.path.exists(model_path): - huggingface.HF_download_file( - predictors_url + predictor, - model_path - ) - - return os.path.exists(model_path) - - def download_embedder(embedders_mode, hubert): - model_path = ( - os.path.join( - configs["speaker_diarization_path"], - "models", - hubert - ) - ) if embedders_mode == "whisper" else ( - os.path.join( - configs["embedders_path"], - hubert - ) - ) - - if embedders_mode != "transformers" and not os.path.exists(model_path): - if embedders_mode == "whisper": - huggingface.HF_download_file( - "".join([ - codecs.decode( - "uggcf://uhttvatsnpr.pb/NauC/Ivrganzrfr-EIP-Cebwrpg/erfbyir/znva/fcrnxre_qvnevmngvba/", - "rot13" - ), - hubert - ]), - model_path - ) - else: - huggingface.HF_download_file( - "".join([ - embedders_url, "fairseq/" if embedders_mode == "fairseq" else "onnx/", - hubert - ]), - model_path - ) - elif embedders_mode == "transformers": - bin_file = os.path.join(model_path, "model.safetensors") - config_file = os.path.join(model_path, "config.json") - - os.makedirs(model_path, exist_ok=True) - - if not os.path.exists(bin_file): - huggingface.HF_download_file( - "".join([embedders_url, "transformers/", hubert, "/model.safetensors"]), - bin_file - ) - - if not os.path.exists(config_file): - huggingface.HF_download_file( - "".join([embedders_url, "transformers/", hubert, "/config.json"]), - config_file - ) - - return os.path.exists(bin_file) and os.path.exists(config_file) - - return os.path.exists(model_path) - - def get_modelname(f0_method, predictor_onnx=False): - suffix = ".onnx" if predictor_onnx else (".pt" if "crepe" not in f0_method else ".pth") - - if "rmvpe" in f0_method: - modelname = ( - "hpa-rmvpe-76000" - if "previous" in f0_method else - "hpa-rmvpe-112000" - ) if "hpa" in f0_method else ( - "rmvpe-mix" - if "mix" in f0_method else - "rmvpe" - ) - elif "fcpe" in f0_method: - modelname = ( - "fcpe_legacy" - if "legacy" in f0_method else - "fcpe" - ) if "previous" in f0_method or "legacy" in f0_method else "ddsp_200k" - elif "crepe" in f0_method: - modelname = "crepe_" + f0_method.replace("mangio-", "").split("-")[1] - elif "penn" in f0_method: - modelname = "fcn" - elif "djcm" in f0_method: - modelname = "djcm" + ("-svs" if "svs" in f0_method else "") - elif "pesto" in f0_method: - modelname = "pesto" - elif "swift" in f0_method: - return "swift.onnx" - else: - return None - - return modelname + suffix - - results = [] - count = configs.get("num_of_restart", 5) - - for _ in range(count): - if "hybrid" in f0_method: - methods_str = re.search(r"hybrid\[(.+)\]", f0_method) - - if methods_str: - methods = [ - f0_method.strip() - for f0_method in methods_str.group(1).split("+") - ] - - for method in methods: - modelname = get_modelname(method, predictor_onnx) - - if modelname is not None: - results.append( - download_predictor(modelname) - ) - else: - modelname = get_modelname(f0_method, predictor_onnx) - - if modelname is not None: - results.append( - download_predictor(modelname) - ) - - if hubert in embedders_model + whisper_model: - if embedders_mode != "transformers": hubert += ".pt" if embedders_mode in ["fairseq", "whisper"] else ".onnx" - - results.append( - download_embedder( - embedders_mode, - hubert - ) - ) - - if all(results): return - else: results = [] - - logger.warning(translations["check_assets_error"].format(count=count)) - sys.exit(1) - -def check_spk_diarization(model_size, speechbrain=True): - whisper_model = os.path.join(configs["speaker_diarization_path"], "models", f"{model_size}.pt") - - if not os.path.exists(whisper_model) and model_size is not None: - huggingface.HF_download_file( - "".join([ - codecs.decode( - "uggcf://uhttvatsnpr.pb/NauC/Ivrganzrfr-EIP-Cebwrpg/erfbyir/znva/fcrnxre_qvnevmngvba/", - "rot13" - ), - model_size, - ".pt" - ]), - whisper_model - ) - - speechbrain_path = os.path.join(configs["speaker_diarization_path"], "models", "speechbrain") - if not os.path.exists(speechbrain_path): os.makedirs(speechbrain_path, exist_ok=True) - - if speechbrain: - for f in [ - "classifier.ckpt", - "config.json", - "embedding_model.ckpt", - "hyperparams.yaml", - "mean_var_norm_emb.ckpt" - ]: - speechbrain_model = os.path.join(speechbrain_path, f) - - if not os.path.exists(speechbrain_model): - huggingface.HF_download_file( - codecs.decode( - "uggcf://uhttvatsnpr.pb/NauC/Ivrganzrfr-EIP-Cebwrpg/erfbyir/znva/fcrnxre_qvnevmngvba/fcrrpuoenva/", - "rot13" - ) + f, - speechbrain_model - ) - -def check_upscaler(): - upscaler_model = os.path.join("assets", "models", "upscalers", "upscalers.pth") - if not os.path.exists(upscaler_model): - huggingface.HF_download_file( - codecs.decode( - "uggcf://uhttvatsnpr.pb/NauC/Ivrganzrfr-EIP-Cebwrpg/erfbyir/znva/hcfpnyref/hcfpnyref.cgu", - "rot13" - ), - upscaler_model - ) - -def load_embedders_model(embedder_model, embedders_mode="fairseq"): - if embedders_mode in ["fairseq", "whisper"] and not embedder_model.endswith(".pt"): embedder_model += ".pt" - elif embedders_mode == "onnx" and not embedder_model.endswith(".onnx"): embedder_model += ".onnx" - - if os.path.exists(embedder_model): embedder_model_path = embedder_model - else: - embedder_model_path = ( - os.path.join( - configs["speaker_diarization_path"], - "models", - embedder_model - ) - ) if embedders_mode == "whisper" else ( - os.path.join( - configs["embedders_path"], - embedder_model - ) - ) - - if not os.path.exists(embedder_model_path): - raise FileNotFoundError( - f"{translations['not_found'].format(name=translations['model'])}: {embedder_model}" - ) - - try: - if embedders_mode == "fairseq": - from main.library.embedders.fairseq import load_model - - hubert_model = load_model( - embedder_model_path - ) - elif embedders_mode == "onnx": - from main.library.embedders.onnx import HubertModelONNX - - hubert_model = HubertModelONNX( - embedder_model_path, - config.providers - ) - elif embedders_mode == "transformers": - from main.library.embedders.transformers import HubertModelWithFinalProj - - hubert_model = HubertModelWithFinalProj.from_pretrained( - embedder_model_path - ) - elif embedders_mode == "whisper": - from main.library.embedders.ppg import WhisperModel - - hubert_model = WhisperModel( - embedder_model_path - ) - else: raise ValueError(translations["option_not_valid"]) - except Exception as e: - raise RuntimeError(translations["read_model_error"].format(e=e)) - - return hubert_model - -def extract_features(model, feats, version, mix=False, mix_layers=9, mix_ratio=0.5): - if hasattr(model, "_finalproj"): model._finalproj = version == "v1" - - logits = model.extract_features( - **{ - "source": feats, - "output_layer": 9 if version == "v1" else 12 - } - )[0] - - if mix: - mix_logits = model.extract_features( - **{ - "source": feats, - "output_layer": mix_layers - } - )[0] - - logits = mix_ratio * mix_logits + (1 - mix_ratio) * logits - - feats = model.final_proj(logits) if version == "v1" else logits - - return feats - -def clear_gpu_cache(): - gc.collect() - - if config.device.startswith("cuda"): torch.cuda.empty_cache() - elif config.device.startswith("xpu"): torch.xpu.empty_cache() - elif config.device.startswith("mps"): torch.mps.empty_cache() - elif config.device.startswith("privateuseone"): directml.empty_cache() - elif config.device.startswith("ocl"): opencl.pytorch_ocl.empty_cache() - -def circular_write(new_data, target): - offset = new_data.shape[0] - - target[: -offset] = target[offset :].detach().clone() - target[-offset :] = new_data - - return target - -def load_faiss_index(index_path, nprobe=1): - index = IndexWrapper(index_path, nprobe=nprobe, device=config.device, is_half=config.is_half, faiss_cpu=configs.get("faiss_cpu", False) or config.device.startswith(("privateuseone", "ocl"))) - big_npy, _ = index.read_index_tensor() - - return index, big_npy - -def load_model(model_path, weights_only=True, log_severity_level=3): - if not os.path.isfile(model_path): return None - - if model_path.endswith(".pth"): - return torch.load( - model_path, - map_location="cpu", - weights_only=weights_only - ) - else: - from main.library.onnx.wrapper import ONNXRVC - - return ONNXRVC( - model_path, - config.providers, - log_severity_level=log_severity_level - ) - -def strtobool(value): - value = value.lower() - - if value in ('y', 'yes', 't', 'true', 'on', '1'): return 1 - elif value in ('n', 'no', 'f', 'false', 'off', '0'): return 0 - else: raise ValueError - -def check_ffmpeg(): - try: - if not sys.platform == "win32": return - ffmpeg_url = codecs.decode("uggcf://uhttvatsnpr.pb/NauC/Ivrganzrfr-EIP-Cebwrpg/erfbyir/znva/sszcrt/", "rot13") - - if not os.path.exists("ffmpeg.exe"): huggingface.HF_download_file(ffmpeg_url + "ffmpeg.exe", output_path="ffmpeg.exe") - if not os.path.exists("ffprobe.exe"): huggingface.HF_download_file(ffmpeg_url + "ffprobe.exe", output_path="ffprobe.exe") - except: - logger.error(translations["ffmpeg_error"]) - return - -if configs.get("ffmpeg_download", True): check_ffmpeg() \ No newline at end of file diff --git a/main/library/uvr5_lib/common_separator.py b/main/library/uvr5_lib/common_separator.py deleted file mode 100644 index d58cf662097cc4b7a2567028e9d0bdcc2ad55128..0000000000000000000000000000000000000000 --- a/main/library/uvr5_lib/common_separator.py +++ /dev/null @@ -1,149 +0,0 @@ -import os -import sys -import librosa - -import numpy as np -import soundfile as sf - -sys.path.append(os.getcwd()) - -from main.library.uvr5_lib.spec_utils import normalize - -class CommonSeparator: - VOCAL_STEM = "Vocals" - INST_STEM = "Instrumental" - OTHER_STEM = "Other" - BASS_STEM = "Bass" - DRUM_STEM = "Drums" - GUITAR_STEM = "Guitar" - PIANO_STEM = "Piano" - SYNTH_STEM = "Synthesizer" - STRINGS_STEM = "Strings" - WOODWINDS_STEM = "Woodwinds" - BRASS_STEM = "Brass" - WIND_INST_STEM = "Wind Inst" - PRIMARY_STEM = "Primary Stem" - SECONDARY_STEM = "Secondary Stem" - LEAD_VOCAL_STEM = "lead_only" - BV_VOCAL_STEM = "backing_only" - NO_STEM = "No " - - STEM_PAIR_MAPPER = { - VOCAL_STEM: INST_STEM, - INST_STEM: VOCAL_STEM, - LEAD_VOCAL_STEM: BV_VOCAL_STEM, - BV_VOCAL_STEM: LEAD_VOCAL_STEM, - PRIMARY_STEM: SECONDARY_STEM - } - - NON_ACCOM_STEMS = ( - VOCAL_STEM, - OTHER_STEM, - BASS_STEM, - DRUM_STEM, - GUITAR_STEM, - PIANO_STEM, - SYNTH_STEM, - STRINGS_STEM, - WOODWINDS_STEM, - BRASS_STEM, - WIND_INST_STEM - ) - - def __init__(self, config): - self.logger = config.get("logger") - self.torch_device = config.get("torch_device") - self.torch_device_cpu = config.get("torch_device_cpu") - self.torch_device_mps = config.get("torch_device_mps") - self.onnx_execution_provider = config.get("onnx_execution_provider") - self.model_name = config.get("model_name") - self.model_path = config.get("model_path") - self.model_data = config.get("model_data") - self.output_dir = config.get("output_dir") - self.output_format = config.get("output_format") - self.output_bitrate = config.get("output_bitrate") - self.normalization_threshold = config.get("normalization_threshold") - self.enable_denoise = config.get("enable_denoise") - self.output_single_stem = config.get("output_single_stem") - self.invert_using_spec = config.get("invert_using_spec") - self.sample_rate = config.get("sample_rate") - self.primary_stem_name = None - self.secondary_stem_name = None - - if "training" in self.model_data and "instruments" in self.model_data["training"]: - instruments = self.model_data["training"]["instruments"] - if instruments: - self.primary_stem_name = instruments[0] - self.secondary_stem_name = instruments[1] if len(instruments) > 1 else self.secondary_stem(self.primary_stem_name) - - if self.primary_stem_name is None: - self.primary_stem_name = self.model_data.get("primary_stem", "Vocals") - self.secondary_stem_name = self.secondary_stem(self.primary_stem_name) - - self.is_karaoke = self.model_data.get("is_karaoke", False) - self.is_bv_model = self.model_data.get("is_bv_model", False) - self.bv_model_rebalance = self.model_data.get("is_bv_model_rebalanced", 0) - self.audio_file_path = None - self.audio_file_base = None - self.primary_source = None - self.secondary_source = None - self.primary_stem_output_path = None - self.secondary_stem_output_path = None - self.cached_sources_map = {} - - def secondary_stem(self, primary_stem): - primary_stem = primary_stem if primary_stem else self.NO_STEM - return self.STEM_PAIR_MAPPER[primary_stem] if primary_stem in self.STEM_PAIR_MAPPER else primary_stem.replace(self.NO_STEM, "") if self.NO_STEM in primary_stem else f"{self.NO_STEM}{primary_stem}" - - def separate(self, audio_file_path): - pass - - def final_process(self, stem_path, source, stem_name): - self.write_audio_soundfile(stem_path, source) - return {stem_name: source} - - def cached_sources_clear(self): - self.cached_sources_map = {} - - def cached_source_callback(self, model_architecture, model_name=None): - model, sources = None, None - mapper = self.cached_sources_map[model_architecture] - for key, value in mapper.items(): - if model_name in key: - model = key - sources = value - - return model, sources - - def cached_model_source_holder(self, model_architecture, sources, model_name=None): - self.cached_sources_map[model_architecture] = {**self.cached_sources_map.get(model_architecture, {}), **{model_name: sources}} - - def prepare_mix(self, mix): - if not isinstance(mix, np.ndarray): - mix, _ = librosa.load(mix, mono=False, sr=self.sample_rate) - else: - mix = mix.T - - if mix.ndim == 1: - mix = np.asfortranarray([mix, mix]) - - return mix - - def write_audio_soundfile(self, stem_path, stem_source): - stem_source = normalize(wave=stem_source, max_peak=self.normalization_threshold) - if np.max(np.abs(stem_source)) < 1e-6: return - - if self.output_dir: - os.makedirs(self.output_dir, exist_ok=True) - stem_path = os.path.join(self.output_dir, stem_path) - - if stem_source.shape[1] == 2 and stem_source.flags["F_CONTIGUOUS"]: stem_source = np.ascontiguousarray(stem_source) - sf.write(stem_path, stem_source, self.sample_rate) - - def clear_file_specific_paths(self): - self.audio_file_path = None - self.audio_file_base = None - self.primary_source = None - self.secondary_source = None - self.primary_stem_output_path = None - self.secondary_stem_output_path = None \ No newline at end of file diff --git a/main/library/uvr5_lib/demucs/apply.py b/main/library/uvr5_lib/demucs/apply.py deleted file mode 100644 index aacec2ac1094bef6a5c8bf1ae385f9734449a11a..0000000000000000000000000000000000000000 --- a/main/library/uvr5_lib/demucs/apply.py +++ /dev/null @@ -1,219 +0,0 @@ -import os -import sys -import tqdm -import torch -import random -import concurrent.futures - -sys.path.append(os.getcwd()) - -from main.library.uvr5_lib.demucs.utils import center_trim - -class DummyPoolExecutor: - class DummyResult: - def __init__( - self, - func, - *args, - **kwargs - ): - self.func = func - self.args = args - self.kwargs = kwargs - - def result(self): - return self.func( - *self.args, - **self.kwargs - ) - - def __init__(self, workers=0): - pass - - def submit(self, func, *args, **kwargs): - return DummyPoolExecutor.DummyResult(func, *args, **kwargs) - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, exc_tb): - return - -class TensorChunk: - def __init__( - self, - tensor, - offset=0, - length=None - ): - total_length = tensor.shape[-1] - assert offset >= 0 - assert offset < total_length - - length = total_length - offset if length is None else min(total_length - offset, length) - - if isinstance(tensor, TensorChunk): - self.tensor = tensor.tensor - self.offset = offset + tensor.offset - else: - self.tensor = tensor - self.offset = offset - - self.length = length - self.device = tensor.device - - @property - def shape(self): - shape = list(self.tensor.shape) - shape[-1] = self.length - return shape - - def padded(self, target_length): - delta = target_length - self.length - total_length = self.tensor.shape[-1] - assert delta >= 0 - - start = self.offset - delta // 2 - end = start + target_length - - correct_start = max(0, start) - correct_end = min(total_length, end) - - pad_left = correct_start - start - pad_right = end - correct_end - - out = torch.nn.functional.pad( - self.tensor[..., correct_start:correct_end], - (pad_left, pad_right) - ) - - assert out.shape[-1] == target_length - return out - -def tensor_chunk(tensor_or_chunk): - if isinstance(tensor_or_chunk, TensorChunk): return tensor_or_chunk - else: - assert isinstance(tensor_or_chunk, torch.Tensor) - return TensorChunk(tensor_or_chunk) - -def apply_model( - model, - mix, - shifts=1, - split=True, - overlap=0.25, - transition_power=1.0, - static_shifts=1, - set_progress_bar=None, - device=None, - progress=False, - num_workers=0, - pool=None, - hybrid = False, - weights = None -): - global fut_length, bag_num, prog_bar - - device = mix.device if device is None else torch.device(device) - if pool is None: pool = concurrent.futures.ThreadPoolExecutor(num_workers) if num_workers > 0 and device.type == "cpu" else DummyPoolExecutor() - - kwargs = { - "shifts": shifts, - "split": split, - "overlap": overlap, - "transition_power": transition_power, - "progress": progress, - "device": device, - "pool": pool, - "set_progress_bar": set_progress_bar, - "static_shifts": static_shifts - } - - if hybrid: - estimates, fut_length, prog_bar, current_model = 0, 0, 0, 0 - totals = [0] * len(model[0].sources) - bag_num = len(model) - - for sub_model, weight in zip(model, weights): - original_model_device = next(iter(sub_model.parameters())).device - sub_model.to(device).eval() - fut_length += fut_length - current_model += 1 - out = apply_model(sub_model, mix, **kwargs) - sub_model.to(original_model_device) - - for k, inst_weight in enumerate(weight): - out[:, k, :, :] *= inst_weight - totals[k] += inst_weight - - estimates += out - del out - - for k in range(estimates.shape[1]): - estimates[:, k, :, :] /= totals[k] - - return estimates - - assert transition_power >= 1 - batch, channels, length = mix.shape - - if shifts: - kwargs["shifts"] = 0 - max_shift = int(0.5 * model.samplerate) - mix = tensor_chunk(mix) - padded_mix = mix.padded(length + 2 * max_shift) - out = 0 - - for _ in range(shifts): - offset = random.randint(0, max_shift) - shifted = TensorChunk(padded_mix, offset, length + max_shift - offset) - shifted_out = apply_model(model, shifted, **kwargs) - out += shifted_out[..., max_shift - offset :] - - out /= shifts - return out - elif split: - kwargs["split"] = False - out = torch.zeros(batch, len(model.sources), channels, length, device=mix.device) - sum_weight = torch.zeros(length, device=mix.device) - segment = int(model.samplerate * model.segment) - stride = int((1 - overlap) * segment) - offsets = range(0, length, stride) - weight = torch.cat([torch.arange(1, segment // 2 + 1, device=device), torch.arange(segment - segment // 2, 0, -1, device=device)]) - assert len(weight) == segment - weight = (weight / weight.max()) ** transition_power - futures = [] - - for offset in offsets: - chunk = TensorChunk(mix, offset, segment) - future = pool.submit(apply_model, model, chunk, **kwargs) - futures.append((future, offset)) - offset += segment - - if progress: futures = tqdm.tqdm(futures) - - for future, offset in futures: - if set_progress_bar: - fut_length = len(futures) * bag_num * static_shifts - prog_bar += 1 - set_progress_bar(0.1, (0.8 / fut_length * prog_bar)) - - chunk_out = future.result() - chunk_length = chunk_out.shape[-1] - - out[..., offset : offset + segment] += (weight[:chunk_length].to(device) * chunk_out).to(mix.device) - sum_weight[offset : offset + segment] += weight[:chunk_length].to(mix.device) - - assert sum_weight.min() > 0 - - out /= sum_weight - return out - else: - valid_length = model.valid_length(length) if hasattr(model, "valid_length") else length - mix = tensor_chunk(mix) - padded_mix = mix.padded(valid_length).to(device) - - with torch.no_grad(): - out = model(padded_mix) - - return center_trim(out, length) \ No newline at end of file diff --git a/main/library/uvr5_lib/demucs/demucs.py b/main/library/uvr5_lib/demucs/demucs.py deleted file mode 100644 index d8f34f1e694a998c89612c107c2248597747cf1d..0000000000000000000000000000000000000000 --- a/main/library/uvr5_lib/demucs/demucs.py +++ /dev/null @@ -1,538 +0,0 @@ -import os -import sys -import math -import torch -import inspect -import functools - -import torch.nn as nn -import torch.nn.functional as F - -sys.path.append(os.getcwd()) - -from main.library.uvr5_lib.demucs.utils import center_trim - -def capture_init(init): - @functools.wraps(init) - def __init__(self, *args, **kwargs): - self._init_args_kwargs = (args, kwargs) - init(self, *args, **kwargs) - - return __init__ - -def unfold(a, kernel_size, stride): - *shape, length = a.shape - n_frames = math.ceil(length / stride) - tgt_length = (n_frames - 1) * stride + kernel_size - a = F.pad(a, (0, tgt_length - length)) - strides = list(a.stride()) - assert strides[-1] == 1 - strides = strides[:-1] + [stride, 1] - return a.as_strided([*shape, n_frames, kernel_size], strides) - -def rescale_conv(conv, reference): - scale = (conv.weight.std().detach() / reference) ** 0.5 - conv.weight.data /= scale - if conv.bias is not None: conv.bias.data /= scale - -def rescale_module(module, reference): - for sub in module.modules(): - if isinstance(sub, (nn.Conv1d, nn.ConvTranspose1d, nn.Conv2d, nn.ConvTranspose2d)): rescale_conv(sub, reference) - -class BLSTM(nn.Module): - def __init__( - self, - dim, - layers=1, - max_steps=None, - skip=False - ): - super().__init__() - assert max_steps is None or max_steps % 4 == 0 - self.max_steps = max_steps - self.lstm = nn.LSTM( - bidirectional=True, - num_layers=layers, - hidden_size=dim, - input_size=dim - ) - self.linear = nn.Linear(2 * dim, dim) - self.skip = skip - - def forward(self, x): - B, C, T = x.shape - y = x - framed = False - - if self.max_steps is not None and T > self.max_steps: - width = self.max_steps - stride = width // 2 - frames = unfold(x, width, stride) - nframes = frames.shape[2] - framed = True - x = frames.permute(0, 2, 1, 3).reshape(-1, C, width) - - x = x.permute(2, 0, 1) - x = self.lstm(x)[0] - x = self.linear(x) - x = x.permute(1, 2, 0) - - if framed: - out = [] - frames = x.reshape(B, -1, C, width) - limit = stride // 2 - - for k in range(nframes): - if k == 0: out.append(frames[:, k, :, :-limit]) - elif k == nframes - 1: out.append(frames[:, k, :, limit:]) - else: out.append(frames[:, k, :, limit:-limit]) - - out = torch.cat(out, -1) - out = out[..., :T] - x = out - - if self.skip: x = x + y - return x - -class LayerScale(nn.Module): - def __init__( - self, - channels, - init = 0 - ): - super().__init__() - self.scale = nn.Parameter( - torch.zeros(channels, requires_grad=True) - ) - self.scale.data[:] = init - - def forward(self, x): - return self.scale[:, None] * x - -class DConv(nn.Module): - def __init__( - self, - channels, - compress = 4, - depth = 2, - init = 1e-4, - norm=True, - attn=False, - heads=4, - ndecay=4, - lstm=False, - gelu=True, - kernel=3, - dilate=True - ): - super().__init__() - assert kernel % 2 == 1 - self.channels = channels - self.compress = compress - self.depth = abs(depth) - dilate = depth > 0 - norm_fn = lambda d: nn.Identity() - if norm: norm_fn = lambda d: nn.GroupNorm(1, d) - hidden = int(channels / compress) - act = nn.GELU if gelu else nn.ReLU - self.layers = nn.ModuleList([]) - - for d in range(self.depth): - dilation = 2**d if dilate else 1 - padding = dilation * (kernel // 2) - - mods = [ - nn.Conv1d( - channels, - hidden, - kernel, - dilation=dilation, - padding=padding - ), - norm_fn(hidden), - act(), - nn.Conv1d(hidden, 2 * channels, 1), - norm_fn(2 * channels), - nn.GLU(1), - LayerScale(channels, init) - ] - - if attn: mods.insert(3, LocalState(hidden, heads=heads, ndecay=ndecay)) - if lstm: mods.insert(3, BLSTM(hidden, layers=2, max_steps=200, skip=True)) - - layer = nn.Sequential(*mods) - self.layers.append(layer) - - def forward(self, x): - for layer in self.layers: - x = x + layer(x) - - return x - -class LocalState(nn.Module): - def __init__( - self, - channels, - heads = 4, - nfreqs = 0, - ndecay = 4 - ): - super().__init__() - assert channels % heads == 0, (channels, heads) - self.heads = heads - self.nfreqs = nfreqs - self.ndecay = ndecay - self.content = nn.Conv1d(channels, channels, 1) - self.query = nn.Conv1d(channels, channels, 1) - self.key = nn.Conv1d(channels, channels, 1) - - if nfreqs: self.query_freqs = nn.Conv1d(channels, heads * nfreqs, 1) - - if ndecay: - self.query_decay = nn.Conv1d(channels, heads * ndecay, 1) - self.query_decay.weight.data *= 0.01 - assert self.query_decay.bias is not None - self.query_decay.bias.data[:] = -2 - - self.proj = nn.Conv1d(channels + heads * nfreqs, channels, 1) - - def forward(self, x): - B, C, T = x.shape - heads = self.heads - indexes = torch.arange(T, device=x.device, dtype=x.dtype) - delta = indexes[:, None] - indexes[None, :] - queries = self.query(x).view(B, heads, -1, T) - keys = self.key(x).view(B, heads, -1, T) - dots = torch.einsum("bhct,bhcs->bhts", keys, queries) - dots /= keys.shape[2] ** 0.5 - - if self.nfreqs: - periods = torch.arange(1, self.nfreqs + 1, device=x.device, dtype=x.dtype) - freq_kernel = (2 * math.pi * delta / periods.view(-1, 1, 1)).cos() - freq_q = self.query_freqs(x).view(B, heads, -1, T) / self.nfreqs**0.5 - dots += torch.einsum("fts,bhfs->bhts", freq_kernel, freq_q) - - if self.ndecay: - decays = torch.arange(1, self.ndecay + 1, device=x.device, dtype=x.dtype) - decay_q = self.query_decay(x).view(B, heads, -1, T) - decay_q = decay_q.sigmoid() / 2 - decay_kernel = -decays.view(-1, 1, 1) * delta.abs() / self.ndecay**0.5 - dots += torch.einsum("fts,bhfs->bhts", decay_kernel, decay_q) - - dots.masked_fill_(torch.eye(T, device=dots.device, dtype=torch.bool), -100) - weights = dots.softmax(dim=2) - content = self.content(x).view(B, heads, -1, T) - result = torch.einsum("bhts,bhct->bhcs", weights, content) - - if self.nfreqs: - time_sig = torch.einsum("bhts,fts->bhfs", weights, freq_kernel) - result = torch.cat([result, time_sig], 2) - - result = result.reshape(B, -1, T) - return x + self.proj(result) - -class Demucs(nn.Module): - @capture_init - def __init__( - self, - sources, - audio_channels=2, - channels=64, - growth=2.0, - depth=6, - rewrite=True, - lstm_layers=0, - kernel_size=8, - stride=4, - context=1, - gelu=True, - glu=True, - norm_starts=4, - norm_groups=4, - dconv_mode=1, - dconv_depth=2, - dconv_comp=4, - dconv_attn=4, - dconv_lstm=4, - dconv_init=1e-4, - normalize=True, - resample=True, - rescale=0.1, - samplerate=44100, - segment=4 * 10 - ): - super().__init__() - self.audio_channels = audio_channels - self.sources = sources - self.kernel_size = kernel_size - self.context = context - self.stride = stride - self.depth = depth - self.resample = resample - self.channels = channels - self.normalize = normalize - self.samplerate = samplerate - self.segment = segment - self.encoder = nn.ModuleList() - self.decoder = nn.ModuleList() - self.skip_scales = nn.ModuleList() - - if glu: - activation = nn.GLU(dim=1) - ch_scale = 2 - else: - activation = nn.ReLU() - ch_scale = 1 - - act2 = nn.GELU if gelu else nn.ReLU - - in_channels = audio_channels - padding = 0 - - for index in range(depth): - norm_fn = lambda d: nn.Identity() - if index >= norm_starts: norm_fn = lambda d: nn.GroupNorm(norm_groups, d) - - encode = [] - encode += [nn.Conv1d(in_channels, channels, kernel_size, stride), norm_fn(channels), act2()] - attn = index >= dconv_attn - lstm = index >= dconv_lstm - - if dconv_mode & 1: - encode += [ - DConv( - channels, - depth=dconv_depth, - init=dconv_init, - compress=dconv_comp, - attn=attn, lstm=lstm - ) - ] - - if rewrite: - encode += [ - nn.Conv1d( - channels, - ch_scale * channels, - 1 - ), - norm_fn(ch_scale * channels), - activation - ] - - self.encoder.append(nn.Sequential(*encode)) - - decode = [] - out_channels = in_channels if index > 0 else len(self.sources) * audio_channels - - if rewrite: - decode += [ - nn.Conv1d( - channels, - ch_scale * channels, - 2 * context + 1, - padding=context - ), - norm_fn(ch_scale * channels), - activation - ] - - if dconv_mode & 2: - decode += [ - DConv( - channels, - depth=dconv_depth, - init=dconv_init, - compress=dconv_comp, - attn=attn, - lstm=lstm - ) - ] - - decode += [ - nn.ConvTranspose1d( - channels, - out_channels, - kernel_size, - stride, - padding=padding - ) - ] - - if index > 0: decode += [norm_fn(out_channels), act2()] - self.decoder.insert(0, nn.Sequential(*decode)) - - in_channels = channels - channels = int(growth * channels) - - channels = in_channels - self.lstm = BLSTM(channels, lstm_layers) if lstm_layers else None - if rescale: rescale_module(self, reference=rescale) - - def valid_length(self, length): - if self.resample: length *= 2 - - for _ in range(self.depth): - length = math.ceil((length - self.kernel_size) / self.stride) + 1 - length = max(1, length) - - for _ in range(self.depth): - length = (length - 1) * self.stride + self.kernel_size - - if self.resample: length = math.ceil(length / 2) - return int(length) - - def forward(self, mix): - x = mix - length = x.shape[-1] - - if self.normalize: - mono = mix.mean(dim=1, keepdim=True) - mean = mono.mean(dim=-1, keepdim=True) - std = mono.std(dim=-1, keepdim=True) - x = (x - mean) / (1e-5 + std) - else: - mean = 0 - std = 1 - - delta = self.valid_length(length) - length - x = F.pad(x, (delta // 2, delta - delta // 2)) - - if self.resample: x = resample_frac(x, 1, 2) - saved = [] - - for encode in self.encoder: - x = encode(x) - saved.append(x) - - if self.lstm: x = self.lstm(x) - - for decode in self.decoder: - skip = saved.pop(-1) - skip = center_trim(skip, x) - x = decode(x + skip) - - if self.resample: x = resample_frac(x, 2, 1) - - x = x * std + mean - x = center_trim(x, length) - x = x.view(x.size(0), len(self.sources), self.audio_channels, x.size(-1)) - - return x - - def load_state_dict(self, state, strict=True): - for idx in range(self.depth): - for a in ["encoder", "decoder"]: - for b in ["bias", "weight"]: - new = f"{a}.{idx}.3.{b}" - old = f"{a}.{idx}.2.{b}" - - if old in state and new not in state: state[new] = state.pop(old) - super().load_state_dict(state, strict=strict) - -class ResampleFrac(torch.nn.Module): - def __init__( - self, - old_sr, - new_sr, - zeros = 24, - rolloff = 0.945 - ): - super().__init__() - gcd = math.gcd(old_sr, new_sr) - self.old_sr = old_sr // gcd - self.new_sr = new_sr // gcd - self.zeros = zeros - self.rolloff = rolloff - self._init_kernels() - - def _init_kernels(self): - if self.old_sr == self.new_sr: return - - kernels = [] - sr = min(self.new_sr, self.old_sr) - sr *= self.rolloff - - self._width = math.ceil(self.zeros * self.old_sr / sr) - idx = torch.arange(-self._width, self._width + self.old_sr).float() - - for i in range(self.new_sr): - t = ((-i / self.new_sr + idx / self.old_sr) * sr).clamp_(-self.zeros, self.zeros) - t *= math.pi - - kernel = sinc(t) * ((t / self.zeros / 2).cos()**2) - kernel.div_(kernel.sum()) - kernels.append(kernel) - - self.register_buffer("kernel", torch.stack(kernels).view(self.new_sr, 1, -1)) - - def forward(self, x, output_length = None, full = False): - if self.old_sr == self.new_sr: return x - shape = x.shape - length = x.shape[-1] - - x = x.reshape(-1, length) - y = F.conv1d( - F.pad( - x[:, None], - (self._width, self._width + self.old_sr), - mode='replicate' - ), - self.kernel, - stride=self.old_sr - ).transpose(1, 2).reshape(list(shape[:-1]) + [-1]) - - float_output_length = torch.as_tensor(self.new_sr * length / self.old_sr) - max_output_length = float_output_length.ceil().long() - default_output_length = float_output_length.floor().long() - - if output_length is None: applied_output_length = max_output_length if full else default_output_length - elif output_length < 0 or output_length > max_output_length: - raise ValueError - else: - applied_output_length = torch.tensor(output_length) - if full: raise ValueError - - return y[..., :applied_output_length] - - def __repr__(self): - return simple_repr(self) - -def sinc(x): - return torch.where(x == 0, torch.tensor(1., device=x.device, dtype=x.dtype), x.sin() / x) - -def simple_repr(obj, attrs = None, overrides = {}): - params = inspect.signature(obj.__class__).parameters - attrs_repr = [] - - if attrs is None: attrs = list(params.keys()) - for attr in attrs: - display = False - - if attr in overrides: value = overrides[attr] - elif hasattr(obj, attr): value = getattr(obj, attr) - else: continue - - if attr in params: - param = params[attr] - if param.default is inspect._empty or value != param.default: display = True - else: display = True - - if display: attrs_repr.append(f"{attr}={value}") - return f"{obj.__class__.__name__}({','.join(attrs_repr)})" - -def resample_frac( - x, - old_sr, - new_sr, - zeros = 24, - rolloff = 0.945, - output_length = None, - full = False -): - return ResampleFrac( - old_sr, - new_sr, - zeros, - rolloff - ).to(x)(x, output_length, full) \ No newline at end of file diff --git a/main/library/uvr5_lib/demucs/hdemucs.py b/main/library/uvr5_lib/demucs/hdemucs.py deleted file mode 100644 index 789d2537041d2667d384a3b2ba4ffccead6a8612..0000000000000000000000000000000000000000 --- a/main/library/uvr5_lib/demucs/hdemucs.py +++ /dev/null @@ -1,1005 +0,0 @@ -import os -import sys -import math -import torch -import functools - -from torch import nn -from copy import deepcopy - -from torch.nn import functional as F - -sys.path.append(os.getcwd()) - -from main.library.uvr5_lib.demucs.demucs import DConv, rescale_module - -def capture_init(init): - @functools.wraps(init) - def __init__(self, *args, **kwargs): - self._init_args_kwargs = (args, kwargs) - init(self, *args, **kwargs) - - return __init__ - -def spectro(x, n_fft=512, hop_length=None, pad=0): - *other, length = x.shape - x = x.reshape(-1, length) - device_type = x.device.type - - is_other_gpu = not device_type in ["cuda", "xpu", "cpu"] - if is_other_gpu: x = x.cpu() - - z = torch.stft( - x, - n_fft=n_fft * (1 + pad), - hop_length=hop_length or n_fft // 4, - window=torch.hann_window(n_fft).to(x), - win_length=n_fft, - normalized=True, - center=True, - return_complex=True, - pad_mode="reflect" - ) - - _, freqs, frame = z.shape - return z.view(*other, freqs, frame) - -def ispectro(z, hop_length=None, length=None, pad=0): - *other, freqs, frames = z.shape - n_fft = 2 * freqs - 2 - z = z.view(-1, freqs, frames) - - win_length = n_fft // (1 + pad) - device_type = z.device.type - - is_other_gpu = not device_type in ["cuda", "xpu", "cpu"] - if is_other_gpu: z = z.cpu() - - x = torch.istft( - z, - n_fft, - hop_length, - window=torch.hann_window(win_length).to(z.real), - win_length=win_length, - normalized=True, - length=length, - center=True - ) - - _, length = x.shape - return x.view(*other, length) - -def atan2(y, x): - pi = 2 * torch.asin(torch.tensor(1.0)) - x += ((x == 0) & (y == 0)) * 1.0 - out = torch.atan(y / x) - out += ((y >= 0) & (x < 0)) * pi - out -= ((y < 0) & (x < 0)) * pi - out *= 1 - ((y > 0) & (x == 0)) * 1.0 - out += ((y > 0) & (x == 0)) * (pi / 2) - out *= 1 - ((y < 0) & (x == 0)) * 1.0 - out += ((y < 0) & (x == 0)) * (-pi / 2) - return out - -def _norm(x): - return x[..., 0].abs() ** 2 + x[..., 1].abs() ** 2 - -def _mul_add(a, b, out = None): - target_shape = torch.Size([max(sa, sb) for (sa, sb) in zip(a.shape, b.shape)]) - - if out is None or out.shape != target_shape: - out = torch.zeros( - target_shape, - dtype=a.dtype, - device=a.device - ) - - if out is a: - real_a = a[..., 0] - out[..., 0] = out[..., 0] + (real_a * b[..., 0] - a[..., 1] * b[..., 1]) - out[..., 1] = out[..., 1] + (real_a * b[..., 1] + a[..., 1] * b[..., 0]) - else: - out[..., 0] = out[..., 0] + (a[..., 0] * b[..., 0] - a[..., 1] * b[..., 1]) - out[..., 1] = out[..., 1] + (a[..., 0] * b[..., 1] + a[..., 1] * b[..., 0]) - - return out - -def _mul(a, b, out = None): - target_shape = torch.Size([max(sa, sb) for (sa, sb) in zip(a.shape, b.shape)]) - - if out is None or out.shape != target_shape: - out = torch.zeros( - target_shape, - dtype=a.dtype, - device=a.device - ) - - if out is a: - real_a = a[..., 0] - out[..., 0] = real_a * b[..., 0] - a[..., 1] * b[..., 1] - out[..., 1] = real_a * b[..., 1] + a[..., 1] * b[..., 0] - else: - out[..., 0] = a[..., 0] * b[..., 0] - a[..., 1] * b[..., 1] - out[..., 1] = a[..., 0] * b[..., 1] + a[..., 1] * b[..., 0] - - return out - -def _inv(z, out = None): - ez = _norm(z) - if out is None or out.shape != z.shape: out = torch.zeros_like(z) - - out[..., 0] = z[..., 0] / ez - out[..., 1] = -z[..., 1] / ez - - return out - -def _conj(z, out = None): - if out is None or out.shape != z.shape: out = torch.zeros_like(z) - - out[..., 0] = z[..., 0] - out[..., 1] = -z[..., 1] - - return out - -def _invert(M, out = None): - nb_channels = M.shape[-2] - if out is None or out.shape != M.shape: out = torch.empty_like(M) - - if nb_channels == 1: out = _inv(M, out) - elif nb_channels == 2: - det = _mul(M[..., 0, 0, :], M[..., 1, 1, :]) - det = det - _mul(M[..., 0, 1, :], M[..., 1, 0, :]) - invDet = _inv(det) - out[..., 0, 0, :] = _mul(invDet, M[..., 1, 1, :], out[..., 0, 0, :]) - out[..., 1, 0, :] = _mul(-invDet, M[..., 1, 0, :], out[..., 1, 0, :]) - out[..., 0, 1, :] = _mul(-invDet, M[..., 0, 1, :], out[..., 0, 1, :]) - out[..., 1, 1, :] = _mul(invDet, M[..., 0, 0, :], out[..., 1, 1, :]) - else: raise Exception - return out - -def expectation_maximization(y, x, iterations = 2, eps = 1e-10, batch_size = 200): - (nb_frames, nb_bins, nb_channels) = x.shape[:-1] - nb_sources = y.shape[-1] - - regularization = torch.cat(( - torch.eye(nb_channels, dtype=x.dtype, device=x.device)[..., None], - torch.zeros((nb_channels, nb_channels, 1), dtype=x.dtype, device=x.device) - ), dim=2) - - regularization = ( - torch.as_tensor(eps) - ).sqrt() * ( - regularization[None, None, ...].expand((-1, nb_bins, -1, -1, -1)) - ) - - R = [ - torch.zeros((nb_bins, nb_channels, nb_channels, 2), dtype=x.dtype, device=x.device) - for j in range(nb_sources) - ] - - weight = torch.zeros((nb_bins,), dtype=x.dtype, device=x.device) - v = torch.zeros((nb_frames, nb_bins, nb_sources), dtype=x.dtype, device=x.device) - - for _ in range(iterations): - v = torch.mean(y[..., 0, :].abs() ** 2 + y[..., 1, :].abs() ** 2, dim=-2) - for j in range(nb_sources): - R[j] = torch.tensor(0.0, device=x.device) - - weight = torch.tensor(eps, device=x.device) - pos = 0 - batch_size = batch_size if batch_size else nb_frames - - while pos < nb_frames: - t = torch.arange(pos, min(nb_frames, pos + batch_size)) - pos = int(t[-1]) + 1 - - R[j] = R[j] + _covariance(y[t, ..., j]).sum(dim=0) - weight = weight + v[t, ..., j].sum(dim=0) - - R[j] = R[j] / weight[..., None, None, None] - weight = torch.zeros_like(weight) - - if y.requires_grad: y = y.clone() - - pos = 0 - - while pos < nb_frames: - t = torch.arange(pos, min(nb_frames, pos + batch_size)) - pos = int(t[-1]) + 1 - - y[t, ...] = torch.tensor(0.0, device=x.device, dtype=x.dtype) - - Cxx = regularization - - for j in range(nb_sources): - Cxx = Cxx + (v[t, ..., j, None, None, None] * R[j][None, ...].clone()) - - inv_Cxx = _invert(Cxx) - - for j in range(nb_sources): - gain = torch.zeros_like(inv_Cxx) - - indices = torch.cartesian_prod( - torch.arange(nb_channels), - torch.arange(nb_channels), - torch.arange(nb_channels) - ) - - for index in indices: - gain[:, :, index[0], index[1], :] = _mul_add( - R[j][None, :, index[0], index[2], :].clone(), - inv_Cxx[:, :, index[2], index[1], :], - gain[:, :, index[0], index[1], :] - ) - - gain = gain * v[t, ..., None, None, None, j] - - for i in range(nb_channels): - y[t, ..., j] = _mul_add( - gain[..., i, :], - x[t, ..., i, None, :], - y[t, ..., j] - ) - - return y, v, R - -def wiener( - targets_spectrograms, - mix_stft, - iterations = 1, - softmask = False, - residual = False, - scale_factor = 10.0, - eps = 1e-10 -): - if softmask: - y = mix_stft[..., None] * ( - targets_spectrograms / ( - eps + targets_spectrograms.sum(dim=-1, keepdim=True).to(mix_stft.dtype) - ) - )[..., None, :] - else: - angle = atan2(mix_stft[..., 1], mix_stft[..., 0])[..., None] - nb_sources = targets_spectrograms.shape[-1] - - y = torch.zeros(mix_stft.shape + (nb_sources,), dtype=mix_stft.dtype, device=mix_stft.device) - y[..., 0, :] = targets_spectrograms * angle.cos() - y[..., 1, :] = targets_spectrograms * angle.sin() - - if residual: y = torch.cat([y, mix_stft[..., None] - y.sum(dim=-1, keepdim=True)], dim=-1) - if iterations == 0: return y - - max_abs = torch.as_tensor( - 1.0, - dtype=mix_stft.dtype, - device=mix_stft.device - ).max(_norm(mix_stft).sqrt().max() / scale_factor) - - mix_stft = mix_stft / max_abs - y = y / max_abs - y = expectation_maximization(y, mix_stft, iterations, eps=eps)[0] - y = y * max_abs - - return y - -def _covariance(y_j): - (nb_frames, nb_bins, nb_channels) = y_j.shape[:-1] - - Cj = torch.zeros( - (nb_frames, nb_bins, nb_channels, nb_channels, 2), - dtype=y_j.dtype, - device=y_j.device - ) - - indices = torch.cartesian_prod( - torch.arange(nb_channels), - torch.arange(nb_channels) - ) - - for index in indices: - Cj[:, :, index[0], index[1], :] = _mul_add( - y_j[:, :, index[0], :], - _conj(y_j[:, :, index[1], :]), - Cj[:, :, index[0], index[1], :] - ) - - return Cj - -def pad1d(x, paddings, mode = "constant", value = 0.0): - x0 = x - length = x.shape[-1] - padding_left, padding_right = paddings - - if mode == "reflect": - max_pad = max(padding_left, padding_right) - - if length <= max_pad: - extra_pad = max_pad - length + 1 - extra_pad_right = min(padding_right, extra_pad) - extra_pad_left = extra_pad - extra_pad_right - paddings = (padding_left - extra_pad_left, padding_right - extra_pad_right) - x = F.pad(x, (extra_pad_left, extra_pad_right)) - - out = F.pad(x, paddings, mode, value) - - assert out.shape[-1] == length + padding_left + padding_right - assert (out[..., padding_left : padding_left + length] == x0).all() - return out - -class ScaledEmbedding(nn.Module): - def __init__( - self, - num_embeddings, - embedding_dim, - scale = 10.0, - smooth=False - ): - super().__init__() - self.embedding = nn.Embedding( - num_embeddings, - embedding_dim - ) - - if smooth: - weight = self.embedding.weight.data.cumsum(dim=0) - weight = weight / torch.arange(1, num_embeddings + 1).to(weight).sqrt()[:, None] - self.embedding.weight.data[:] = weight - - self.embedding.weight.data /= scale - self.scale = scale - - @property - def weight(self): - return self.embedding.weight * self.scale - - def forward(self, x): - return self.embedding(x) * self.scale - -class HEncLayer(nn.Module): - def __init__( - self, - chin, - chout, - kernel_size=8, - stride=4, - norm_groups=1, - empty=False, - freq=True, - dconv=True, - norm=True, - context=0, - dconv_kw={}, - pad=True, - rewrite=True - ): - super().__init__() - norm_fn = lambda d: nn.Identity() - if norm: norm_fn = lambda d: nn.GroupNorm(norm_groups, d) - pad = kernel_size // 4 if pad else 0 - - klass = nn.Conv1d - self.freq = freq - self.kernel_size = kernel_size - self.stride = stride - self.empty = empty - self.norm = norm - self.pad = pad - - if freq: - kernel_size = [kernel_size, 1] - stride = [stride, 1] - pad = [pad, 0] - klass = nn.Conv2d - - self.conv = klass(chin, chout, kernel_size, stride, pad) - if self.empty: return - - self.norm1 = norm_fn(chout) - self.rewrite = None - - if rewrite: - self.rewrite = klass(chout, 2 * chout, 1 + 2 * context, 1, context) - self.norm2 = norm_fn(2 * chout) - - self.dconv = None - if dconv: self.dconv = DConv(chout, **dconv_kw) - - def forward(self, x, inject=None): - if not self.freq and x.dim() == 4: - B, C, Fr, T = x.shape - x = x.view(B, -1, T) - - if not self.freq: - le = x.shape[-1] - if not le % self.stride == 0: x = F.pad(x, (0, self.stride - (le % self.stride))) - - y = self.conv(x) - if self.empty: return y - - if inject is not None: - assert inject.shape[-1] == y.shape[-1], (inject.shape, y.shape) - - if inject.dim() == 3 and y.dim() == 4: inject = inject[:, :, None] - y = y + inject - - y = F.gelu(self.norm1(y)) - - if self.dconv: - if self.freq: - B, C, Fr, T = y.shape - y = y.permute(0, 2, 1, 3).reshape(-1, C, T) - - y = self.dconv(y) - if self.freq: y = y.view(B, Fr, C, T).permute(0, 2, 1, 3) - - if self.rewrite: - z = self.norm2(self.rewrite(y)) - z = F.glu(z, dim=1) - else: z = y - - return z - -class MultiWrap(nn.Module): - def __init__( - self, - layer, - split_ratios - ): - super().__init__() - self.split_ratios = split_ratios - self.layers = nn.ModuleList() - self.conv = isinstance(layer, HEncLayer) - assert not layer.norm - assert layer.freq - assert layer.pad - - if not self.conv: assert not layer.context_freq - - for _ in range(len(split_ratios) + 1): - lay = deepcopy(layer) - - if self.conv: lay.conv.padding = (0, 0) - else: lay.pad = False - - for m in lay.modules(): - if hasattr(m, "reset_parameters"): m.reset_parameters() - - self.layers.append(lay) - - def forward(self, x, skip=None, length=None): - B, C, Fr, T = x.shape - ratios = list(self.split_ratios) + [1] - start = 0 - outs = [] - - for ratio, layer in zip(ratios, self.layers): - if self.conv: - pad = layer.kernel_size // 4 - - if ratio == 1: - limit = Fr - frames = -1 - else: - limit = int(round(Fr * ratio)) - le = limit - start - - if start == 0: le += pad - - frames = round((le - layer.kernel_size) / layer.stride + 1) - limit = start + (frames - 1) * layer.stride + layer.kernel_size - - if start == 0: limit -= pad - - assert limit - start > 0, (limit, start) - assert limit <= Fr, (limit, Fr) - - y = x[:, :, start:limit, :] - - if start == 0: y = F.pad(y, (0, 0, pad, 0)) - if ratio == 1: y = F.pad(y, (0, 0, 0, pad)) - - outs.append(layer(y)) - start = limit - layer.kernel_size + layer.stride - else: - limit = Fr if ratio == 1 else int(round(Fr * ratio)) - - last = layer.last - layer.last = True - - y = x[:, :, start:limit] - s = skip[:, :, start:limit] - out, _ = layer(y, s, None) - - if outs: - outs[-1][:, :, -layer.stride :] += out[:, :, : layer.stride] - layer.conv_tr.bias.view(1, -1, 1, 1) - out = out[:, :, layer.stride :] - - if ratio == 1: out = out[:, :, : -layer.stride // 2, :] - if start == 0: out = out[:, :, layer.stride // 2 :, :] - - outs.append(out) - layer.last = last - start = limit - - out = torch.cat(outs, dim=2) - if not self.conv and not last: out = F.gelu(out) - - if self.conv: return out - else: return out, None - -class HDecLayer(nn.Module): - def __init__( - self, - chin, - chout, - last=False, - kernel_size=8, - stride=4, - norm_groups=1, - empty=False, - freq=True, - dconv=True, - norm=True, - context=1, - dconv_kw={}, - pad=True, - context_freq=True, - rewrite=True - ): - super().__init__() - norm_fn = lambda d: nn.Identity() - - if norm: norm_fn = lambda d: nn.GroupNorm(norm_groups, d) - pad = kernel_size // 4 if pad else 0 - - self.pad = pad - self.last = last - self.freq = freq - self.chin = chin - self.empty = empty - self.stride = stride - self.kernel_size = kernel_size - self.norm = norm - self.context_freq = context_freq - klass = nn.Conv1d - klass_tr = nn.ConvTranspose1d - - if freq: - kernel_size = [kernel_size, 1] - stride = [stride, 1] - klass = nn.Conv2d - klass_tr = nn.ConvTranspose2d - - self.conv_tr = klass_tr(chin, chout, kernel_size, stride) - self.norm2 = norm_fn(chout) - - if self.empty: return - self.rewrite = None - - if rewrite: - if context_freq: self.rewrite = klass(chin, 2 * chin, 1 + 2 * context, 1, context) - else: self.rewrite = klass(chin, 2 * chin, [1, 1 + 2 * context], 1, [0, context]) - - self.norm1 = norm_fn(2 * chin) - - self.dconv = None - if dconv: self.dconv = DConv(chin, **dconv_kw) - - def forward(self, x, skip, length): - if self.freq and x.dim() == 3: - B, C, T = x.shape - x = x.view(B, self.chin, -1, T) - - if not self.empty: - x = x + skip - - y = F.glu(self.norm1(self.rewrite(x)), dim=1) if self.rewrite else x - - if self.dconv: - if self.freq: - B, C, Fr, T = y.shape - y = y.permute(0, 2, 1, 3).reshape(-1, C, T) - - y = self.dconv(y) - if self.freq: y = y.view(B, Fr, C, T).permute(0, 2, 1, 3) - else: - y = x - assert skip is None - - z = self.norm2(self.conv_tr(y)) - - if self.freq: - if self.pad: z = z[..., self.pad : -self.pad, :] - else: - z = z[..., self.pad : self.pad + length] - assert z.shape[-1] == length, (z.shape[-1], length) - - if not self.last: z = F.gelu(z) - return z, y - -class HDemucs(nn.Module): - @capture_init - def __init__( - self, - sources, - audio_channels=2, - channels=48, - channels_time=None, - growth=2, nfft=4096, - wiener_iters=0, - end_iters=0, - wiener_residual=False, - cac=True, - depth=6, - rewrite=True, - hybrid=True, - hybrid_old=False, - multi_freqs=None, - multi_freqs_depth=2, - freq_emb=0.2, - emb_scale=10, - emb_smooth=True, - kernel_size=8, - time_stride=2, - stride=4, - context=1, - context_enc=0, - norm_starts=4, - norm_groups=4, - dconv_mode=1, - dconv_depth=2, - dconv_comp=4, - dconv_attn=4, - dconv_lstm=4, - dconv_init=1e-4, - rescale=0.1, - samplerate=44100, - segment=4 * 10 - ): - super().__init__() - self.cac = cac - self.wiener_residual = wiener_residual - self.audio_channels = audio_channels - self.sources = sources - self.kernel_size = kernel_size - self.context = context - self.stride = stride - self.depth = depth - self.channels = channels - self.samplerate = samplerate - self.segment = segment - self.nfft = nfft - self.hop_length = nfft // 4 - self.wiener_iters = wiener_iters - self.end_iters = end_iters - self.freq_emb = None - self.hybrid = hybrid - self.hybrid_old = hybrid_old - if hybrid_old: assert hybrid - if hybrid: assert wiener_iters == end_iters - self.encoder = nn.ModuleList() - self.decoder = nn.ModuleList() - - if hybrid: - self.tencoder = nn.ModuleList() - self.tdecoder = nn.ModuleList() - - chin = audio_channels - chin_z = chin - - if self.cac: chin_z *= 2 - - chout = channels_time or channels - chout_z = channels - freqs = nfft // 2 - - for index in range(depth): - lstm = index >= dconv_lstm - attn = index >= dconv_attn - norm = index >= norm_starts - freq = freqs > 1 - stri = stride - ker = kernel_size - - if not freq: - assert freqs == 1 - - ker = time_stride * 2 - stri = time_stride - - pad = True - last_freq = False - - if freq and freqs <= kernel_size: - ker = freqs - pad = False - last_freq = True - - kw = { - "kernel_size": ker, - "stride": stri, - "freq": freq, - "pad": pad, - "norm": norm, - "rewrite": rewrite, - "norm_groups": norm_groups, - "dconv_kw": { - "lstm": lstm, - "attn": attn, - "depth": dconv_depth, - "compress": dconv_comp, - "init": dconv_init, - "gelu": True - }, - } - - kwt = dict(kw) - kwt["freq"] = 0 - kwt["kernel_size"] = kernel_size - kwt["stride"] = stride - kwt["pad"] = True - kw_dec = dict(kw) - - multi = False - - if multi_freqs and index < multi_freqs_depth: - multi = True - kw_dec["context_freq"] = False - - if last_freq: - chout_z = max(chout, chout_z) - chout = chout_z - - enc = HEncLayer( - chin_z, - chout_z, - dconv=dconv_mode & 1, - context=context_enc, - **kw - ) - - if hybrid and freq: - tenc = HEncLayer( - chin, - chout, - dconv=dconv_mode & 1, - context=context_enc, - empty=last_freq, - **kwt - ) - self.tencoder.append(tenc) - - if multi: - enc = MultiWrap( - enc, - multi_freqs - ) - - self.encoder.append(enc) - if index == 0: - chin = self.audio_channels * len(self.sources) - chin_z = chin - - if self.cac: chin_z *= 2 - - dec = HDecLayer( - chout_z, - chin_z, - dconv=dconv_mode & 2, - last=index == 0, - context=context, - **kw_dec - ) - - if multi: - dec = MultiWrap( - dec, - multi_freqs - ) - - if hybrid and freq: - tdec = HDecLayer( - chout, chin, - dconv=dconv_mode & 2, - empty=last_freq, - last=index == 0, - context=context, - **kwt - ) - - self.tdecoder.insert(0, tdec) - - self.decoder.insert(0, dec) - chin = chout - chin_z = chout_z - chout = int(growth * chout) - chout_z = int(growth * chout_z) - - if freq: - if freqs <= kernel_size: freqs = 1 - else: freqs //= stride - - if index == 0 and freq_emb: - self.freq_emb = ScaledEmbedding( - freqs, - chin_z, - smooth=emb_smooth, - scale=emb_scale - ) - - self.freq_emb_scale = freq_emb - - if rescale: rescale_module(self, reference=rescale) - - def _spec(self, x): - hl = self.hop_length - nfft = self.nfft - - if self.hybrid: - assert hl == nfft // 4 - le = int(math.ceil(x.shape[-1] / hl)) - pad = hl // 2 * 3 - x = pad1d( - x, - (pad, pad + le * hl - x.shape[-1]), - mode="reflect" - ) if not self.hybrid_old else pad1d( - x, - (pad, pad + le * hl - x.shape[-1]) - ) - - z = spectro(x, nfft, hl)[..., :-1, :] - if self.hybrid: - assert z.shape[-1] == le + 4, (z.shape, x.shape, le) - z = z[..., 2 : 2 + le] - - return z - - def _ispec(self, z, length=None, scale=0): - hl = self.hop_length // (4**scale) - z = F.pad(z, (0, 0, 0, 1)) - - if self.hybrid: - z = F.pad(z, (2, 2)) - pad = hl // 2 * 3 - le = hl * int(math.ceil(length / hl)) + 2 * pad if not self.hybrid_old else hl * int(math.ceil(length / hl)) - x = ispectro(z, hl, length=le) - x = x[..., pad : pad + length] if not self.hybrid_old else x[..., :length] - else: x = ispectro(z, hl, length) - - return x - - def _magnitude(self, z): - if self.cac: - B, C, Fr, T = z.shape - m = torch.view_as_real(z).permute(0, 1, 4, 2, 3) - m = m.reshape(B, C * 2, Fr, T) - else: m = z.abs() - - return m - - def _mask(self, z, m): - niters = self.wiener_iters - if self.cac: - B, S, C, Fr, T = m.shape - out = m.view(B, S, -1, 2, Fr, T).permute(0, 1, 2, 4, 5, 3) - out = torch.view_as_complex(out.contiguous()) - return out - - if self.training: niters = self.end_iters - - if niters < 0: - z = z[:, None] - return z / (1e-8 + z.abs()) * m - else: return self._wiener(m, z, niters) - - def _wiener(self, mag_out, mix_stft, niters): - init = mix_stft.dtype - wiener_win_len = 300 - residual = self.wiener_residual - B, S, C, Fq, T = mag_out.shape - mag_out = mag_out.permute(0, 4, 3, 2, 1) - mix_stft = torch.view_as_real(mix_stft.permute(0, 3, 2, 1)) - outs = [] - - for sample in range(B): - pos = 0 - out = [] - - for pos in range(0, T, wiener_win_len): - frame = slice(pos, pos + wiener_win_len) - z_out = wiener(mag_out[sample, frame], mix_stft[sample, frame], niters, residual=residual) - out.append(z_out.transpose(-1, -2)) - - outs.append(torch.cat(out, dim=0)) - - out = torch.view_as_complex(torch.stack(outs, 0)) - out = out.permute(0, 4, 3, 2, 1).contiguous() - - if residual: out = out[:, :-1] - assert list(out.shape) == [B, S, C, Fq, T] - return out.to(init) - - def forward(self, mix): - x = mix - length = x.shape[-1] - z = self._spec(mix) - mag = self._magnitude(z).to(mix.device) - x = mag - B, C, Fq, T = x.shape - mean = x.mean(dim=(1, 2, 3), keepdim=True) - std = x.std(dim=(1, 2, 3), keepdim=True) - x = (x - mean) / (1e-5 + std) - - if self.hybrid: - xt = mix - meant = xt.mean(dim=(1, 2), keepdim=True) - stdt = xt.std(dim=(1, 2), keepdim=True) - xt = (xt - meant) / (1e-5 + stdt) - - saved, saved_t, lengths, lengths_t = [], [], [], [] - - for idx, encode in enumerate(self.encoder): - lengths.append(x.shape[-1]) - inject = None - - if self.hybrid and idx < len(self.tencoder): - lengths_t.append(xt.shape[-1]) - tenc = self.tencoder[idx] - xt = tenc(xt) - - if not tenc.empty: saved_t.append(xt) - else: inject = xt - - x = encode(x, inject) - - if idx == 0 and self.freq_emb is not None: - frs = torch.arange(x.shape[-2], device=x.device) - emb = self.freq_emb(frs).t()[None, :, :, None].expand_as(x) - x = x + self.freq_emb_scale * emb - - saved.append(x) - - x = torch.zeros_like(x) - if self.hybrid: xt = torch.zeros_like(x) - - for idx, decode in enumerate(self.decoder): - skip = saved.pop(-1) - x, pre = decode(x, skip, lengths.pop(-1)) - - if self.hybrid: offset = self.depth - len(self.tdecoder) - - if self.hybrid and idx >= offset: - tdec = self.tdecoder[idx - offset] - length_t = lengths_t.pop(-1) - - if tdec.empty: - assert pre.shape[2] == 1, pre.shape - - pre = pre[:, :, 0] - xt, _ = tdec(pre, None, length_t) - else: - skip = saved_t.pop(-1) - xt, _ = tdec(xt, skip, length_t) - - assert len(saved) == 0 - assert len(lengths_t) == 0 - assert len(saved_t) == 0 - - S = len(self.sources) - x = x.view(B, S, -1, Fq, T) - x = x * std[:, None] + mean[:, None] - device_type = x.device.type - device_load = f"{device_type}:{x.device.index}" if not device_type == "mps" else device_type - x_is_other_gpu = not device_type in ["cuda", "xpu", "cpu"] - if x_is_other_gpu: x = x.cpu() - zout = self._mask(z, x) - x = self._ispec(zout, length) - if x_is_other_gpu: x = x.to(device_load) - - if self.hybrid: - xt = xt.view(B, S, -1, length) - xt = xt * stdt[:, None] + meant[:, None] - x = xt + x - - return x \ No newline at end of file diff --git a/main/library/uvr5_lib/demucs/htdemucs.py b/main/library/uvr5_lib/demucs/htdemucs.py deleted file mode 100644 index 2663a31ce18f79145ec964e250f48e444bae5ea8..0000000000000000000000000000000000000000 --- a/main/library/uvr5_lib/demucs/htdemucs.py +++ /dev/null @@ -1,1076 +0,0 @@ -import os -import sys -import math -import torch -import random -import functools - -import numpy as np - -from torch import nn -from einops import rearrange -from fractions import Fraction -from torch.nn import functional as F - -sys.path.append(os.getcwd()) - -from main.app.variables import translations -from main.library.uvr5_lib.demucs.demucs import rescale_module -from main.library.uvr5_lib.demucs.hdemucs import pad1d, spectro, ispectro, wiener, ScaledEmbedding, HEncLayer, MultiWrap, HDecLayer - -def capture_init(init): - @functools.wraps(init) - def __init__(self, *args, **kwargs): - self._init_args_kwargs = (args, kwargs) - init(self, *args, **kwargs) - - return __init__ - -def create_sin_embedding( - length, - dim, - shift = 0, - device="cpu", - max_period=10000 -): - assert dim % 2 == 0 - - pos = shift + torch.arange(length, device=device).view(-1, 1, 1) - half_dim = dim // 2 - adim = torch.arange(dim // 2, device=device).view(1, 1, -1) - - phase = pos / ( - max_period ** (( - adim.to(torch.float32) / torch.tensor(half_dim - 1, dtype=torch.float32, device=device) - ) if str(device).startswith("ocl") else ( - adim / (half_dim - 1) - )) - ) - - return torch.cat([phase.cos(), phase.sin()], dim=-1) - -def create_2d_sin_embedding( - d_model, - height, - width, - device="cpu", - max_period=10000 -): - if d_model % 4 != 0: raise ValueError(translations["dims"].format(dims=d_model)) - - pe = torch.zeros(d_model, height, width) - d_model = int(d_model / 2) - - div_term = (torch.arange(0.0, d_model, 2) * -(math.log(max_period) / d_model)).exp() - pos_w = torch.arange(0.0, width).unsqueeze(1) - pos_h = torch.arange(0.0, height).unsqueeze(1) - - pe[0:d_model:2, :, :] = (pos_w * div_term).sin().transpose(0, 1).unsqueeze(1).repeat(1, height, 1) - pe[1:d_model:2, :, :] = (pos_w * div_term).cos().transpose(0, 1).unsqueeze(1).repeat(1, height, 1) - pe[d_model::2, :, :] = (pos_h * div_term).sin().transpose(0, 1).unsqueeze(2).repeat(1, 1, width) - pe[d_model + 1 :: 2, :, :] = (pos_h * div_term).cos().transpose(0, 1).unsqueeze(2).repeat(1, 1, width) - - return pe[None, :].to(device) - -def create_sin_embedding_cape( - length, - dim, - batch_size, - mean_normalize, - augment, - max_global_shift = 0.0, - max_local_shift = 0.0, - max_scale = 1.0, - device = "cpu", - max_period = 10000.0 -): - assert dim % 2 == 0 - - pos = 1.0 * torch.arange(length).view(-1, 1, 1) - pos = pos.repeat(1, batch_size, 1) - - if mean_normalize: pos -= torch.nanmean(pos, dim=0, keepdim=True) - - if augment: - delta = np.random.uniform( - -max_global_shift, - +max_global_shift, - size=[1, batch_size, 1] - ) - - delta_local = np.random.uniform( - -max_local_shift, - +max_local_shift, - size=[length, batch_size, 1] - ) - - log_lambdas = np.random.uniform( - -np.log(max_scale), - +np.log(max_scale), - size=[1, batch_size, 1] - ) - - pos = (pos + delta + delta_local) * np.exp(log_lambdas) - - pos = pos.to(device) - half_dim = dim // 2 - - adim = torch.arange(dim // 2, device=device).view(1, 1, -1) - phase = pos / ( - max_period ** (( - adim.to(torch.float32) / torch.tensor(half_dim - 1, dtype=torch.float32, device=device) - ) if str(device).startswith("ocl") else ( - adim / (half_dim - 1) - )) - ) - - return torch.cat([phase.cos(), phase.sin()], dim=-1).float() - -class MyGroupNorm(nn.GroupNorm): - def __init__( - self, - *args, - **kwargs - ): - super().__init__( - *args, - **kwargs - ) - - def forward(self, x): - x = x.transpose(1, 2) - return super().forward(x).transpose(1, 2) - -class LayerScale(nn.Module): - def __init__( - self, - channels, - init = 0, - channel_last=False - ): - super().__init__() - self.channel_last = channel_last - self.scale = nn.Parameter(torch.zeros(channels, requires_grad=True)) - self.scale.data[:] = init - - def forward(self, x): - if self.channel_last: return self.scale * x - else: return self.scale[:, None] * x - -class MyTransformerEncoderLayer(nn.TransformerEncoderLayer): - def __init__( - self, - d_model, - nhead, - dim_feedforward=2048, - dropout=0.1, - activation=F.relu, - group_norm=0, - norm_first=False, - norm_out=False, - layer_norm_eps=1e-5, - layer_scale=False, - init_values=1e-4, - device=None, - dtype=None, - sparse=False, - mask_type="diag", - mask_random_seed=42, - sparse_attn_window=500, - global_window=50, - auto_sparsity=False, - sparsity=0.95, - batch_first=False - ): - factory_kwargs = { - "device": device, - "dtype": dtype - } - - super().__init__( - d_model=d_model, - nhead=nhead, - dim_feedforward=dim_feedforward, - dropout=dropout, - activation=activation, - layer_norm_eps=layer_norm_eps, - batch_first=batch_first, - norm_first=norm_first, - device=device, - dtype=dtype - ) - - self.auto_sparsity = auto_sparsity - - if group_norm: - self.norm1 = MyGroupNorm( - int(group_norm), - d_model, - eps=layer_norm_eps, - **factory_kwargs - ) - - self.norm2 = MyGroupNorm( - int(group_norm), - d_model, - eps=layer_norm_eps, - **factory_kwargs - ) - - self.norm_out = None - if self.norm_first & norm_out: - self.norm_out = MyGroupNorm( - num_groups=int(norm_out), - num_channels=d_model - ) - - self.gamma_1 = LayerScale(d_model, init_values, True) if layer_scale else nn.Identity() - self.gamma_2 = LayerScale(d_model, init_values, True) if layer_scale else nn.Identity() - - def forward(self, src, src_mask=None, src_key_padding_mask=None): - x = src - T, B, C = x.shape - - if self.norm_first: - x = x + self.gamma_1( - self._sa_block(self.norm1(x), src_mask, src_key_padding_mask) - ) - - x = x + self.gamma_2( - self._ff_block(self.norm2(x)) - ) - - if self.norm_out: x = self.norm_out(x) - else: - x = self.norm1( - x + self.gamma_1( - self._sa_block(x, src_mask, src_key_padding_mask) - ) - ) - - x = self.norm2( - x + self.gamma_2( - self._ff_block(x) - ) - ) - - return x - -class CrossTransformerEncoder(nn.Module): - def __init__( - self, - dim, - emb = "sin", - hidden_scale = 4.0, - num_heads = 8, - num_layers = 6, - cross_first = False, - dropout = 0.0, - max_positions = 1000, - norm_in = True, - norm_in_group = False, - group_norm = False, - norm_first = False, - norm_out = False, - max_period = 10000.0, - weight_decay = 0.0, - lr = None, - layer_scale = False, - gelu = True, - sin_random_shift = 0, - weight_pos_embed = 1.0, - cape_mean_normalize = True, - cape_augment = True, - cape_glob_loc_scale = [5000.0, 1.0, 1.4], - sparse_self_attn = False, - sparse_cross_attn = False, - mask_type = "diag", - mask_random_seed = 42, - sparse_attn_window = 500, - global_window = 50, - auto_sparsity = False, - sparsity = 0.95 - ): - super().__init__() - assert dim % num_heads == 0 - hidden_dim = int(dim * hidden_scale) - self.num_layers = num_layers - self.classic_parity = int(cross_first) - self.emb = emb - self.max_period = max_period - self.weight_decay = weight_decay - self.weight_pos_embed = weight_pos_embed - self.sin_random_shift = sin_random_shift - - if emb == "cape": - self.cape_mean_normalize = cape_mean_normalize - self.cape_augment = cape_augment - self.cape_glob_loc_scale = cape_glob_loc_scale - - if emb == "scaled": - self.position_embeddings = ScaledEmbedding( - max_positions, - dim, - scale=0.2 - ) - - self.lr = lr - activation = F.gelu if gelu else F.relu - - if norm_in: - self.norm_in = nn.LayerNorm(dim) - self.norm_in_t = nn.LayerNorm(dim) - elif norm_in_group: - self.norm_in = MyGroupNorm(int(norm_in_group), dim) - self.norm_in_t = MyGroupNorm(int(norm_in_group), dim) - else: - self.norm_in = nn.Identity() - self.norm_in_t = nn.Identity() - - self.layers = nn.ModuleList() - self.layers_t = nn.ModuleList() - - kwargs_common = { - "d_model": dim, - "nhead": num_heads, - "dim_feedforward": hidden_dim, - "dropout": dropout, - "activation": activation, - "group_norm": group_norm, - "norm_first": norm_first, - "norm_out": norm_out, - "layer_scale": layer_scale, - "mask_type": mask_type, - "mask_random_seed": mask_random_seed, - "sparse_attn_window": sparse_attn_window, - "global_window": global_window, - "sparsity": sparsity, - "auto_sparsity": auto_sparsity, - "batch_first": True, - } - - kwargs_classic_encoder = dict(kwargs_common) - kwargs_classic_encoder.update({"sparse": sparse_self_attn}) - kwargs_cross_encoder = dict(kwargs_common) - kwargs_cross_encoder.update({"sparse": sparse_cross_attn}) - - for idx in range(num_layers): - if idx % 2 == self.classic_parity: - self.layers.append( - MyTransformerEncoderLayer( - **kwargs_classic_encoder - ) - ) - - self.layers_t.append( - MyTransformerEncoderLayer( - **kwargs_classic_encoder - ) - ) - else: - self.layers.append( - CrossTransformerEncoderLayer( - **kwargs_cross_encoder - ) - ) - - self.layers_t.append( - CrossTransformerEncoderLayer( - **kwargs_cross_encoder - ) - ) - - def forward(self, x, xt): - B, C, Fr, T1 = x.shape - - pos_emb_2d = create_2d_sin_embedding(C, Fr, T1, x.device, self.max_period) - pos_emb_2d = rearrange(pos_emb_2d, "b c fr t1 -> b (t1 fr) c") - - x = rearrange(x, "b c fr t1 -> b (t1 fr) c") - x = self.norm_in(x) - x = x + self.weight_pos_embed * pos_emb_2d - - B, C, T2 = xt.shape - xt = rearrange(xt, "b c t2 -> b t2 c") - - pos_emb = self._get_pos_embedding(T2, B, C, x.device) - pos_emb = rearrange(pos_emb, "t2 b c -> b t2 c") - - xt = self.norm_in_t(xt) - xt = xt + self.weight_pos_embed * pos_emb - - for idx in range(self.num_layers): - if idx % 2 == self.classic_parity: - x = self.layers[idx](x) - xt = self.layers_t[idx](xt) - else: - old_x = x - x = self.layers[idx](x, xt) - xt = self.layers_t[idx](xt, old_x) - - x = rearrange(x, "b (t1 fr) c -> b c fr t1", t1=T1) - xt = rearrange(xt, "b t2 c -> b c t2") - return x, xt - - def _get_pos_embedding(self, T, B, C, device): - if self.emb == "sin": - shift = random.randrange(self.sin_random_shift + 1) - - pos_emb = create_sin_embedding( - T, - C, - shift=shift, - device=device, - max_period=self.max_period - ) - elif self.emb == "cape": - if self.training: - pos_emb = create_sin_embedding_cape( - T, - C, - B, - device=device, - max_period=self.max_period, - mean_normalize=self.cape_mean_normalize, - augment=self.cape_augment, - max_global_shift=self.cape_glob_loc_scale[0], - max_local_shift=self.cape_glob_loc_scale[1], - max_scale=self.cape_glob_loc_scale[2] - ) - else: - pos_emb = create_sin_embedding_cape( - T, - C, - B, - device=device, - max_period=self.max_period, - mean_normalize=self.cape_mean_normalize, - augment=False - ) - elif self.emb == "scaled": - pos = torch.arange(T, device=device) - pos_emb = self.position_embeddings(pos)[:, None] - - return pos_emb - - def make_optim_group(self): - group = {"params": list(self.parameters()), "weight_decay": self.weight_decay} - if self.lr is not None: group["lr"] = self.lr - return group - -class CrossTransformerEncoderLayer(nn.Module): - def __init__( - self, - d_model, - nhead, - dim_feedforward = 2048, - dropout = 0.1, - activation=F.relu, - layer_norm_eps = 1e-5, - layer_scale = False, - init_values = 1e-4, - norm_first = False, - group_norm = False, - norm_out = False, - sparse=False, - mask_type="diag", - mask_random_seed=42, - sparse_attn_window=500, - global_window=50, - sparsity=0.95, - auto_sparsity=None, - device=None, - dtype=None, - batch_first=False - ): - super().__init__() - self.auto_sparsity = auto_sparsity - factory_kwargs = { - "device": device, - "dtype": dtype - } - - self.cross_attn = nn.MultiheadAttention( - d_model, - nhead, - dropout=dropout, - batch_first=batch_first - ) - - self.linear1 = nn.Linear( - d_model, - dim_feedforward, - **factory_kwargs - ) - - self.norm_first = norm_first - self.dropout = nn.Dropout(dropout) - self.linear2 = nn.Linear( - dim_feedforward, - d_model, - **factory_kwargs - ) - - if group_norm: - self.norm1 = MyGroupNorm( - int(group_norm), - d_model, - eps=layer_norm_eps, - **factory_kwargs - ) - - self.norm2 = MyGroupNorm( - int(group_norm), - d_model, - eps=layer_norm_eps, - **factory_kwargs - ) - - self.norm3 = MyGroupNorm( - int(group_norm), - d_model, - eps=layer_norm_eps, - **factory_kwargs - ) - else: - self.norm1 = nn.LayerNorm( - d_model, - eps=layer_norm_eps, - **factory_kwargs - ) - - self.norm2 = nn.LayerNorm( - d_model, - eps=layer_norm_eps, - **factory_kwargs - ) - - self.norm3 = nn.LayerNorm( - d_model, - eps=layer_norm_eps, - **factory_kwargs - ) - - self.norm_out = None - if self.norm_first & norm_out: - self.norm_out = MyGroupNorm( - num_groups=int(norm_out), - num_channels=d_model - ) - - self.gamma_1 = LayerScale(d_model, init_values, True) if layer_scale else nn.Identity() - self.gamma_2 = LayerScale(d_model, init_values, True) if layer_scale else nn.Identity() - self.dropout1 = nn.Dropout(dropout) - self.dropout2 = nn.Dropout(dropout) - - if isinstance(activation, str): self.activation = self._get_activation_fn(activation) - else: self.activation = activation - - def forward(self, q, k, mask=None): - if self.norm_first: - x = q + self.gamma_1(self._ca_block(self.norm1(q), self.norm2(k), mask)) - x = x + self.gamma_2(self._ff_block(self.norm3(x))) - - if self.norm_out: x = self.norm_out(x) - else: - x = self.norm1(q + self.gamma_1(self._ca_block(q, k, mask))) - x = self.norm2(x + self.gamma_2(self._ff_block(x))) - - return x - - def _ca_block(self, q, k, attn_mask=None): - x = self.cross_attn(q, k, k, attn_mask=attn_mask, need_weights=False)[0] - return self.dropout1(x) - - def _ff_block(self, x): - x = self.linear2(self.dropout(self.activation(self.linear1(x)))) - return self.dropout2(x) - - def _get_activation_fn(self, activation): - if activation == "relu": return F.relu - elif activation == "gelu": return F.gelu - raise RuntimeError(translations["activation"].format(activation=activation)) - -class HTDemucs(nn.Module): - @capture_init - def __init__( - self, - sources, - audio_channels=2, - channels=48, - channels_time=None, - growth=2, - nfft=4096, - wiener_iters=0, - end_iters=0, - wiener_residual=False, - cac=True, - depth=4, - rewrite=True, - multi_freqs=None, - multi_freqs_depth=3, - freq_emb=0.2, - emb_scale=10, - emb_smooth=True, - kernel_size=8, - time_stride=2, - stride=4, - context=1, - context_enc=0, - norm_starts=4, - norm_groups=4, - dconv_mode=1, - dconv_depth=2, - dconv_comp=8, - dconv_init=1e-3, - bottom_channels=0, - t_layers=5, - t_emb="sin", - t_hidden_scale=4.0, - t_heads=8, - t_dropout=0.0, - t_max_positions=10000, - t_norm_in=True, - t_norm_in_group=False, - t_group_norm=False, - t_norm_first=True, - t_norm_out=True, - t_max_period=10000.0, - t_weight_decay=0.0, - t_lr=None, - t_layer_scale=True, - t_gelu=True, - t_weight_pos_embed=1.0, - t_sin_random_shift=0, - t_cape_mean_normalize=True, - t_cape_augment=True, - t_cape_glob_loc_scale=[5000.0, 1.0, 1.4], - t_sparse_self_attn=False, - t_sparse_cross_attn=False, - t_mask_type="diag", - t_mask_random_seed=42, - t_sparse_attn_window=500, - t_global_window=100, - t_sparsity=0.95, - t_auto_sparsity=False, - t_cross_first=False, - rescale=0.1, - samplerate=44100, - segment=4 * 10, - use_train_segment=True - ): - super().__init__() - self.cac = cac - self.wiener_residual = wiener_residual - self.audio_channels = audio_channels - self.sources = sources - self.kernel_size = kernel_size - self.context = context - self.stride = stride - self.depth = depth - self.bottom_channels = bottom_channels - self.channels = channels - self.samplerate = samplerate - self.segment = segment - self.use_train_segment = use_train_segment - self.nfft = nfft - self.hop_length = nfft // 4 - self.wiener_iters = wiener_iters - self.end_iters = end_iters - self.freq_emb = None - assert wiener_iters == end_iters - self.encoder = nn.ModuleList() - self.decoder = nn.ModuleList() - self.tencoder = nn.ModuleList() - self.tdecoder = nn.ModuleList() - chin = audio_channels - chin_z = chin - if self.cac: chin_z *= 2 - chout = channels_time or channels - chout_z = channels - freqs = nfft // 2 - - for index in range(depth): - norm = index >= norm_starts - freq = freqs > 1 - stri = stride - ker = kernel_size - - if not freq: - assert freqs == 1 - ker = time_stride * 2 - stri = time_stride - - pad = True - last_freq = False - - if freq and freqs <= kernel_size: - ker = freqs - pad = False - last_freq = True - - kw = { - "kernel_size": ker, - "stride": stri, - "freq": freq, - "pad": pad, - "norm": norm, - "rewrite": rewrite, - "norm_groups": norm_groups, - "dconv_kw": { - "depth": dconv_depth, - "compress": dconv_comp, - "init": dconv_init, - "gelu": True - }, - } - - kwt = dict(kw) - kwt["freq"] = 0 - kwt["kernel_size"] = kernel_size - kwt["stride"] = stride - kwt["pad"] = True - kw_dec = dict(kw) - multi = False - - if multi_freqs and index < multi_freqs_depth: - multi = True - kw_dec["context_freq"] = False - - if last_freq: - chout_z = max(chout, chout_z) - chout = chout_z - - enc = HEncLayer( - chin_z, - chout_z, - dconv=dconv_mode & 1, - context=context_enc, - **kw - ) - - if freq: - tenc = HEncLayer( - chin, - chout, - dconv=dconv_mode & 1, - context=context_enc, - empty=last_freq, - **kwt - ) - - self.tencoder.append(tenc) - - if multi: - enc = MultiWrap( - enc, - multi_freqs - ) - - self.encoder.append(enc) - if index == 0: - chin = self.audio_channels * len(self.sources) - chin_z = chin - if self.cac: chin_z *= 2 - - dec = HDecLayer( - chout_z, - chin_z, - dconv=dconv_mode & 2, - last=index == 0, - context=context, - **kw_dec - ) - - if multi: - dec = MultiWrap( - dec, - multi_freqs - ) - - if freq: - tdec = HDecLayer( - chout, - chin, - dconv=dconv_mode & 2, - empty=last_freq, - last=index == 0, - context=context, - **kwt - ) - - self.tdecoder.insert(0, tdec) - - self.decoder.insert(0, dec) - chin = chout - chin_z = chout_z - chout = int(growth * chout) - chout_z = int(growth * chout_z) - - if freq: - if freqs <= kernel_size: freqs = 1 - else: freqs //= stride - - if index == 0 and freq_emb: - self.freq_emb = ScaledEmbedding( - freqs, - chin_z, - smooth=emb_smooth, - scale=emb_scale - ) - - self.freq_emb_scale = freq_emb - - if rescale: rescale_module(self, reference=rescale) - transformer_channels = channels * growth ** (depth - 1) - - if bottom_channels: - self.channel_upsampler = nn.Conv1d( - transformer_channels, - bottom_channels, - 1 - ) - - self.channel_downsampler = nn.Conv1d( - bottom_channels, - transformer_channels, - 1 - ) - - self.channel_upsampler_t = nn.Conv1d( - transformer_channels, - bottom_channels, - 1 - ) - - self.channel_downsampler_t = nn.Conv1d( - bottom_channels, - transformer_channels, - 1 - ) - - transformer_channels = bottom_channels - - if t_layers > 0: - self.crosstransformer = CrossTransformerEncoder( - dim=transformer_channels, - emb=t_emb, - hidden_scale=t_hidden_scale, - num_heads=t_heads, - num_layers=t_layers, - cross_first=t_cross_first, - dropout=t_dropout, - max_positions=t_max_positions, - norm_in=t_norm_in, - norm_in_group=t_norm_in_group, - group_norm=t_group_norm, - norm_first=t_norm_first, - norm_out=t_norm_out, - max_period=t_max_period, - weight_decay=t_weight_decay, - lr=t_lr, - layer_scale=t_layer_scale, - gelu=t_gelu, - sin_random_shift=t_sin_random_shift, - weight_pos_embed=t_weight_pos_embed, - cape_mean_normalize=t_cape_mean_normalize, - cape_augment=t_cape_augment, - cape_glob_loc_scale=t_cape_glob_loc_scale, - sparse_self_attn=t_sparse_self_attn, - sparse_cross_attn=t_sparse_cross_attn, - mask_type=t_mask_type, - mask_random_seed=t_mask_random_seed, - sparse_attn_window=t_sparse_attn_window, - global_window=t_global_window, - sparsity=t_sparsity, - auto_sparsity=t_auto_sparsity - ) - else: self.crosstransformer = None - - def _spec(self, x): - hl = self.hop_length - nfft = self.nfft - assert hl == nfft // 4 - le = int(math.ceil(x.shape[-1] / hl)) - pad = hl // 2 * 3 - x = pad1d(x, (pad, pad + le * hl - x.shape[-1]), mode="reflect") - z = spectro(x, nfft, hl)[..., :-1, :] - assert z.shape[-1] == le + 4, (z.shape, x.shape, le) - z = z[..., 2 : 2 + le] - return z - - def _ispec(self, z, length=None, scale=0): - hl = self.hop_length // (4**scale) - z = F.pad(z, (0, 0, 0, 1)) - z = F.pad(z, (2, 2)) - pad = hl // 2 * 3 - le = hl * int(math.ceil(length / hl)) + 2 * pad - x = ispectro(z, hl, length=le) - x = x[..., pad : pad + length] - return x - - def _magnitude(self, z): - if self.cac: - B, C, Fr, T = z.shape - m = torch.view_as_real(z).permute(0, 1, 4, 2, 3) - m = m.reshape(B, C * 2, Fr, T) - else: m = z.abs() - return m - - def _mask(self, z, m): - niters = self.wiener_iters - if self.cac: - B, S, C, Fr, T = m.shape - out = m.view(B, S, -1, 2, Fr, T).permute(0, 1, 2, 4, 5, 3) - out = torch.view_as_complex(out.contiguous()) - return out - - if self.training: niters = self.end_iters - - if niters < 0: - z = z[:, None] - return z / (1e-8 + z.abs()) * m - else: return self._wiener(m, z, niters) - - def _wiener(self, mag_out, mix_stft, niters): - init = mix_stft.dtype - wiener_win_len = 300 - residual = self.wiener_residual - B, S, C, Fq, T = mag_out.shape - mag_out = mag_out.permute(0, 4, 3, 2, 1) - mix_stft = torch.view_as_real(mix_stft.permute(0, 3, 2, 1)) - - outs = [] - - for sample in range(B): - pos = 0 - out = [] - - for pos in range(0, T, wiener_win_len): - frame = slice(pos, pos + wiener_win_len) - z_out = wiener(mag_out[sample, frame], mix_stft[sample, frame], niters, residual=residual) - out.append(z_out.transpose(-1, -2)) - - outs.append(torch.cat(out, dim=0)) - - out = torch.view_as_complex(torch.stack(outs, 0)) - out = out.permute(0, 4, 3, 2, 1).contiguous() - - if residual: out = out[:, :-1] - assert list(out.shape) == [B, S, C, Fq, T] - return out.to(init) - - def valid_length(self, length): - if not self.use_train_segment: return length - - training_length = int(self.segment * self.samplerate) - if training_length < length: raise ValueError(translations["length_or_training_length"].format(length=length, training_length=training_length)) - - return training_length - - def forward(self, mix): - length = mix.shape[-1] - length_pre_pad = None - - if self.use_train_segment: - if self.training: self.segment = Fraction(mix.shape[-1], self.samplerate) - else: - training_length = int(self.segment * self.samplerate) - - if mix.shape[-1] < training_length: - length_pre_pad = mix.shape[-1] - mix = F.pad(mix, (0, training_length - length_pre_pad)) - - z = self._spec(mix) - mag = self._magnitude(z).to(mix.device) - x = mag - B, C, Fq, T = x.shape - mean = x.mean(dim=(1, 2, 3), keepdim=True) - std = x.std(dim=(1, 2, 3), keepdim=True) - x = (x - mean) / (1e-5 + std) - xt = mix - meant = xt.mean(dim=(1, 2), keepdim=True) - stdt = xt.std(dim=(1, 2), keepdim=True) - xt = (xt - meant) / (1e-5 + stdt) - - saved, saved_t, lengths, lengths_t = [], [], [], [] - - for idx, encode in enumerate(self.encoder): - lengths.append(x.shape[-1]) - inject = None - - if idx < len(self.tencoder): - lengths_t.append(xt.shape[-1]) - tenc = self.tencoder[idx] - xt = tenc(xt) - - if not tenc.empty: saved_t.append(xt) - else: inject = xt - - x = encode(x, inject) - if idx == 0 and self.freq_emb is not None: - frs = torch.arange(x.shape[-2], device=x.device) - emb = self.freq_emb(frs).t()[None, :, :, None].expand_as(x) - x = x + self.freq_emb_scale * emb - - saved.append(x) - - if self.crosstransformer: - if self.bottom_channels: - b, c, f, t = x.shape - x = rearrange(x, "b c f t-> b c (f t)") - x = self.channel_upsampler(x) - x = rearrange(x, "b c (f t)-> b c f t", f=f) - xt = self.channel_upsampler_t(xt) - - x, xt = self.crosstransformer(x, xt) - - if self.bottom_channels: - x = rearrange(x, "b c f t-> b c (f t)") - x = self.channel_downsampler(x) - x = rearrange(x, "b c (f t)-> b c f t", f=f) - xt = self.channel_downsampler_t(xt) - - for idx, decode in enumerate(self.decoder): - skip = saved.pop(-1) - x, pre = decode(x, skip, lengths.pop(-1)) - offset = self.depth - len(self.tdecoder) - - if idx >= offset: - tdec = self.tdecoder[idx - offset] - length_t = lengths_t.pop(-1) - - if tdec.empty: - assert pre.shape[2] == 1, pre.shape - pre = pre[:, :, 0] - xt, _ = tdec(pre, None, length_t) - else: - skip = saved_t.pop(-1) - xt, _ = tdec(xt, skip, length_t) - - assert len(saved) == 0 - assert len(lengths_t) == 0 - assert len(saved_t) == 0 - - S = len(self.sources) - x = x.view(B, S, -1, Fq, T) - x = x * std[:, None] + mean[:, None] - device_type = x.device.type - device_load = f"{device_type}:{x.device.index}" if not device_type == "mps" else device_type - x_is_other_gpu = not device_type in ["cuda", "xpu", "cpu"] - if x_is_other_gpu: x = x.cpu() - zout = self._mask(z, x) - - if self.use_train_segment: x = self._ispec(zout, length) if self.training else self._ispec(zout, training_length) - else: x = self._ispec(zout, length) - - if x_is_other_gpu: x = x.to(device_load) - - if self.use_train_segment: xt = xt.view(B, S, -1, length) if self.training else xt.view(B, S, -1, training_length) - else: xt = xt.view(B, S, -1, length) - - xt = xt * stdt[:, None] + meant[:, None] - x = xt + x - - if length_pre_pad: x = x[..., :length_pre_pad] - return x \ No newline at end of file diff --git a/main/library/uvr5_lib/demucs/utils.py b/main/library/uvr5_lib/demucs/utils.py deleted file mode 100644 index f51a10d0c137cf0644442126c8c20065e902ded4..0000000000000000000000000000000000000000 --- a/main/library/uvr5_lib/demucs/utils.py +++ /dev/null @@ -1,10 +0,0 @@ -import torch - -def center_trim(tensor, reference): - ref_size = reference.size(-1) if isinstance(reference, torch.Tensor) else reference - delta = tensor.size(-1) - ref_size - - if delta < 0: raise ValueError - if delta: tensor = tensor[..., delta // 2 : -(delta - delta // 2)] - - return tensor \ No newline at end of file diff --git a/main/library/uvr5_lib/demucs_separator.py b/main/library/uvr5_lib/demucs_separator.py deleted file mode 100644 index 2524406347355be257b7e33fde3c6419e5710872..0000000000000000000000000000000000000000 --- a/main/library/uvr5_lib/demucs_separator.py +++ /dev/null @@ -1,126 +0,0 @@ -import os -import sys -import torch -import warnings -import contextlib - -import numpy as np - -sys.path.append(os.getcwd()) - -from main.app.variables import config -from main.library.utils import clear_gpu_cache -from main.library.uvr5_lib.demucs import apply -from main.library.uvr5_lib import common_separator -from main.library.uvr5_lib.demucs.hdemucs import HDemucs -from main.library.uvr5_lib.demucs.htdemucs import HTDemucs - -if not config.debug_mode: warnings.filterwarnings("ignore") - -DEMUCS_4_SOURCE_MAPPER = { - common_separator.CommonSeparator.BASS_STEM: 0, - common_separator.CommonSeparator.DRUM_STEM: 1, - common_separator.CommonSeparator.OTHER_STEM: 2, - common_separator.CommonSeparator.VOCAL_STEM: 3 -} - -def load_model(modelname, model_path, segment_size = None): - with warnings.catch_warnings() if not config.debug_mode else contextlib.nullcontext(): - if not config.debug_mode: warnings.simplefilter("ignore") - package = torch.load(model_path, map_location="cpu", weights_only=True) - - demucs_fn = HDemucs if modelname.startswith("hdemucs") else HTDemucs - hybrid = False - - if modelname.startswith("htdemucs_ft"): - model_list = [] - for i in range(4): - model = demucs_fn(**package[f"kwargs-{i}"]) - model.load_state_dict(package[f"state-{i}"]) - model.segment = None if segment_size == "Default" or segment_size is None else int(segment_size) - model_list.append(model) - - model = torch.nn.ModuleList(model_list) - hybrid = True - else: - model = demucs_fn(**package["kwargs"]) - model.load_state_dict(package["state"]) - model.segment = None if segment_size == "Default" or segment_size is None else int(segment_size) - - return model, hybrid - -class DemucsSeparator(common_separator.CommonSeparator): - def __init__(self, common_config, arch_config): - super().__init__(config=common_config) - self.segment_size = arch_config.get("segment_size", "Default") - self.shifts = arch_config.get("shifts", 2) - self.overlap = arch_config.get("overlap", 0.25) - self.segments_enabled = arch_config.get("segments_enabled", True) - self.demucs_source_map = DEMUCS_4_SOURCE_MAPPER - if config.configs.get("demucs_cpu_mode", False): self.torch_device = torch.device("cpu") - self.weights = [[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], [0.0, 0.0, 0.0, 1.0]] - self.demucs_model_instance, self.hybrid = load_model(self.model_name, self.model_path, self.segment_size) - self.demucs_model_instance.to(self.torch_device).eval() - - def cleanup(self): - del self.demucs_model_instance - self.demucs_model_instance = None - clear_gpu_cache() - - def separate(self, audio_file_path): - source = self.demix_demucs(audio_file_path) - output_files = [] - - source_length = len(source) - - if source_length == 2: self.demucs_source_map = {common_separator.CommonSeparator.INST_STEM: 0, common_separator.CommonSeparator.VOCAL_STEM: 1} - elif source_length == 6: - self.demucs_source_map = { - common_separator.CommonSeparator.BASS_STEM: 0, - common_separator.CommonSeparator.DRUM_STEM: 1, - common_separator.CommonSeparator.OTHER_STEM: 2, - common_separator.CommonSeparator.VOCAL_STEM: 3, - common_separator.CommonSeparator.GUITAR_STEM: 4, - common_separator.CommonSeparator.PIANO_STEM: 5 - } - else: self.demucs_source_map = DEMUCS_4_SOURCE_MAPPER - - for stem_name, stem_value in self.demucs_source_map.items(): - if self.output_single_stem is not None and stem_name.lower() != self.output_single_stem.lower(): continue - stem_path = os.path.join(f"{os.path.splitext(os.path.basename(audio_file_path))[0]}_({stem_name})_{self.model_name}.{self.output_format.lower()}") - - self.final_process(stem_path, source[stem_value].T, stem_name) - output_files.append(stem_path) - - self.cleanup() - return output_files - - def demix_demucs(self, audio_file_path): - processed = {} - - mix = torch.tensor(self.prepare_mix(audio_file_path), dtype=torch.float32) - ref = mix.mean(0) - - mix = (mix - ref.mean()) / ref.std() - mix_infer = mix - - with torch.inference_mode(): - sources = apply.apply_model( - model=self.demucs_model_instance, - mix=mix_infer[None], - shifts=self.shifts, - split=self.segments_enabled, - overlap=self.overlap, - static_shifts=max(self.shifts, 1), - set_progress_bar=None, - device=self.torch_device, - progress=True, - hybrid=self.hybrid, - weights=self.weights - )[0] - - sources = (sources * ref.std() + ref.mean()).cpu().numpy() - sources[[0, 1]] = sources[[1, 0]] - processed[mix] = sources[:, :, 0:None].copy() - - return np.concatenate([s[:, :, 0:None] for s in list(processed.values())], axis=-1) \ No newline at end of file diff --git a/main/library/uvr5_lib/mdx_separator.py b/main/library/uvr5_lib/mdx_separator.py deleted file mode 100644 index 3129e8661180fc4cc370a16e43770222d37eb9a4..0000000000000000000000000000000000000000 --- a/main/library/uvr5_lib/mdx_separator.py +++ /dev/null @@ -1,271 +0,0 @@ -import os -import sys -import onnx -import torch -import platform -import warnings -import onnx2torch - -import numpy as np -import onnxruntime as ort - -from tqdm import tqdm - -sys.path.append(os.getcwd()) - -from main.app.variables import config -from main.library.uvr5_lib import spec_utils -from main.library.uvr5_lib.common_separator import CommonSeparator - -if not config.debug_mode: warnings.filterwarnings("ignore") - -class MDXSeparator(CommonSeparator): - def __init__(self, common_config, arch_config): - super().__init__(config=common_config) - self.segment_size = arch_config.get("segment_size") - self.overlap = arch_config.get("overlap") - self.batch_size = arch_config.get("batch_size", 1) - self.hop_length = arch_config.get("hop_length") - self.enable_denoise = arch_config.get("enable_denoise") - self.compensate = self.model_data["compensate"] - self.dim_f = self.model_data["mdx_dim_f_set"] - self.dim_t = 2 ** self.model_data["mdx_dim_t_set"] - self.n_fft = self.model_data["mdx_n_fft_scale_set"] - self.config_yaml = self.model_data.get("config_yaml", None) - self.load_model() - self.n_bins = 0 - self.trim = 0 - self.chunk_size = 0 - self.gen_size = 0 - self.stft = None - self.primary_source = None - self.secondary_source = None - self.audio_file_path = None - self.audio_file_base = None - - def load_model(self): - if self.segment_size == self.dim_t: - ort_session_options = ort.SessionOptions() - ort_session_options.log_severity_level = 3 - ort_inference_session = ort.InferenceSession(self.model_path, providers=self.onnx_execution_provider, sess_options=ort_session_options) - self.model_run = lambda spek: ort_inference_session.run(None, {"input": spek.cpu().numpy()})[0] - else: - self.model_run = onnx2torch.convert(onnx.load(self.model_path)) if platform.system() == 'Windows' else onnx2torch.convert(self.model_path) - self.model_run.to(self.torch_device).eval() - - def separate(self, audio_file_path): - self.audio_file_path = audio_file_path - self.audio_file_base = os.path.splitext(os.path.basename(audio_file_path))[0] - mix = self.prepare_mix(self.audio_file_path) - mix = spec_utils.normalize(wave=mix, max_peak=self.normalization_threshold) - peak = np.abs(mix).max() - source = self.demix(mix) * peak - output_files = [] - - if not isinstance(self.primary_source, np.ndarray): self.primary_source = source.T - - if not isinstance(self.secondary_source, np.ndarray): - raw_mix = self.demix(mix, is_match_mix=True) - - if self.invert_using_spec: - self.secondary_source = spec_utils.invert_stem(raw_mix, self.primary_source * self.compensate) - else: - self.secondary_source = (-self.primary_source * self.compensate) + mix.T - - if not self.output_single_stem or self.output_single_stem.lower() == self.secondary_stem_name.lower(): - self.secondary_stem_output_path = os.path.join(f"{self.audio_file_base}_({self.secondary_stem_name})_{self.model_name}.{self.output_format.lower()}") - self.final_process(self.secondary_stem_output_path, self.secondary_source, self.secondary_stem_name) - output_files.append(self.secondary_stem_output_path) - - if not self.output_single_stem or self.output_single_stem.lower() == self.primary_stem_name.lower(): - self.primary_stem_output_path = os.path.join(f"{self.audio_file_base}_({self.primary_stem_name})_{self.model_name}.{self.output_format.lower()}") - - self.final_process(self.primary_stem_output_path, self.primary_source, self.primary_stem_name) - output_files.append(self.primary_stem_output_path) - - return output_files - - def initialize_model_settings(self): - self.n_bins = self.n_fft // 2 + 1 - self.trim = self.n_fft // 2 - self.chunk_size = self.hop_length * (self.segment_size - 1) - self.gen_size = self.chunk_size - 2 * self.trim - self.stft = STFT(self.n_fft, self.hop_length, self.dim_f, self.torch_device) - - def initialize_mix(self, mix, is_ckpt=False): - if is_ckpt: - pad = self.gen_size + self.trim - (mix.shape[-1] % self.gen_size) - mixture = np.concatenate( - ( - np.zeros((2, self.trim), dtype=np.float32), - mix, - np.zeros((2, pad), dtype=np.float32), - np.zeros((2, self.trim), dtype=np.float32), - ), - 1 - ) - - num_chunks = mixture.shape[-1] // self.gen_size - mix_waves = [mixture[:, i * self.gen_size : i * self.gen_size + self.chunk_size] for i in range(num_chunks)] - else: - mix_waves = [] - n_sample = mix.shape[1] - - pad = self.gen_size - n_sample % self.gen_size - mix_p = np.concatenate((np.zeros((2, self.trim)), mix, np.zeros((2, pad)), np.zeros((2, self.trim))), 1) - - i = 0 - while i < n_sample + pad: - mix_waves.append(np.array(mix_p[:, i : i + self.chunk_size])) - i += self.gen_size - - mix_waves_tensor = torch.tensor(mix_waves, dtype=torch.float32).to(self.torch_device) - return mix_waves_tensor, pad - - def demix(self, mix, is_match_mix=False): - self.initialize_model_settings() - tar_waves_ = [] - - if is_match_mix: - chunk_size = self.hop_length * (self.segment_size - 1) - overlap = 0.02 - else: - chunk_size = self.chunk_size - overlap = self.overlap - - gen_size = chunk_size - 2 * self.trim - mixture = np.concatenate((np.zeros((2, self.trim), dtype=np.float32), mix, np.zeros((2, gen_size + self.trim - ((mix.shape[-1]) % gen_size)), dtype=np.float32)), 1) - step = int((1 - overlap) * chunk_size) - - result = np.zeros((1, 2, mixture.shape[-1]), dtype=np.float32) - divider = np.zeros((1, 2, mixture.shape[-1]), dtype=np.float32) - total = 0 - - for i in tqdm(range(0, mixture.shape[-1], step), ncols=100, unit="f"): - total += 1 - start = i - end = min(i + chunk_size, mixture.shape[-1]) - - chunk_size_actual = end - start - window = None - - if overlap != 0: - window = np.hanning(chunk_size_actual) - window = np.tile(window[None, None, :], (1, 2, 1)) - - mix_part_ = mixture[:, start:end] - - if end != i + chunk_size: - pad_size = (i + chunk_size) - end - mix_part_ = np.concatenate((mix_part_, np.zeros((2, pad_size), dtype=np.float32)), axis=-1) - - mix_waves = torch.from_numpy(np.array([mix_part_])).float().to(self.torch_device).split(self.batch_size) - - with torch.no_grad(): - batches_processed = 0 - - for mix_wave in mix_waves: - batches_processed += 1 - tar_waves = self.run_model(mix_wave, is_match_mix=is_match_mix) - - if window is not None: - tar_waves[..., :chunk_size_actual] *= window - divider[..., start:end] += window - else: divider[..., start:end] += 1 - - result[..., start:end] += tar_waves[..., : end - start] - - tar_waves = result / (divider + 1e-8) - tar_waves_.append(tar_waves) - tar_waves = np.concatenate(np.vstack(tar_waves_)[:, :, self.trim : -self.trim], axis=-1)[:, : mix.shape[-1]] - - source = tar_waves[:, 0:None] - return source - - def run_model(self, mix, is_match_mix=False): - spek = self.stft(mix.to(self.torch_device)) - spek[:, :, :3, :] *= 0 - - if is_match_mix: - spec_pred = spek.cpu().numpy() - else: - if self.enable_denoise: - spec_pred_neg = self.model_run(-spek) - spec_pred_pos = self.model_run(spek) - spec_pred = (spec_pred_neg * -0.5) + (spec_pred_pos * 0.5) - else: - spec_pred = self.model_run(spek) - - result = self.stft.inverse((torch.from_numpy(spec_pred) if spec_pred.dtype == np.float32 else spec_pred).to(self.torch_device)).cpu().detach().numpy() - return result - -class STFT: - def __init__(self, n_fft, hop_length, dim_f, device): - self.n_fft = n_fft - self.hop_length = hop_length - self.dim_f = dim_f - self.device = device - self.hann_window = torch.hann_window(window_length=self.n_fft, periodic=True) - - def __call__(self, input_tensor): - is_non_standard_device = not input_tensor.device.type in ["cuda", "xpu", "cpu"] - if is_non_standard_device: input_tensor = input_tensor.cpu() - - batch_dimensions = input_tensor.shape[:-2] - channel_dim, time_dim = input_tensor.shape[-2:] - - permuted_stft_output = torch.view_as_real(torch.stft( - input_tensor.reshape([-1, time_dim]), - n_fft=self.n_fft, - hop_length=self.hop_length, - window=self.hann_window.to(input_tensor.device), - center=True, - return_complex=True - )).permute([0, 3, 1, 2]) - - final_output = permuted_stft_output.reshape([*batch_dimensions, channel_dim, 2, -1, permuted_stft_output.shape[-1]]).reshape([*batch_dimensions, channel_dim * 2, -1, permuted_stft_output.shape[-1]]) - - if is_non_standard_device: final_output = final_output.to(self.device) - return final_output[..., : self.dim_f, :] - - def pad_frequency_dimension(self, input_tensor, batch_dimensions, channel_dim, freq_dim, time_dim, num_freq_bins): - return torch.cat([input_tensor, torch.zeros([*batch_dimensions, channel_dim, num_freq_bins - freq_dim, time_dim]).to(input_tensor.device)], -2) - - def calculate_inverse_dimensions(self, input_tensor): - channel_dim, freq_dim, time_dim = input_tensor.shape[-3:] - - return input_tensor.shape[:-3], channel_dim, freq_dim, time_dim, self.n_fft // 2 + 1 - - def prepare_for_istft(self, padded_tensor, batch_dimensions, channel_dim, num_freq_bins, time_dim): - permuted_tensor = padded_tensor.reshape([*batch_dimensions, channel_dim // 2, 2, num_freq_bins, time_dim]).reshape([-1, 2, num_freq_bins, time_dim]).permute([0, 2, 3, 1]) - - return permuted_tensor[..., 0] + permuted_tensor[..., 1] * 1.0j - - def inverse(self, input_tensor): - is_non_standard_device = not input_tensor.device.type in ["cuda", "xpu", "cpu"] - if is_non_standard_device: input_tensor = input_tensor.cpu() - - batch_dimensions, channel_dim, freq_dim, time_dim, num_freq_bins = self.calculate_inverse_dimensions(input_tensor) - final_output = torch.istft( - self.prepare_for_istft( - self.pad_frequency_dimension( - input_tensor, - batch_dimensions, - channel_dim, - freq_dim, - time_dim, - num_freq_bins - ), - batch_dimensions, - channel_dim, - num_freq_bins, - time_dim - ), - n_fft=self.n_fft, - hop_length=self.hop_length, - window=self.hann_window.to(input_tensor.device), - center=True - ).reshape([*batch_dimensions, 2, -1]) - - if is_non_standard_device: final_output = final_output.to(self.device) - return final_output \ No newline at end of file diff --git a/main/library/uvr5_lib/separator.py b/main/library/uvr5_lib/separator.py deleted file mode 100644 index 569099ad2e8f2466e54d687e450a044d04e275e1..0000000000000000000000000000000000000000 --- a/main/library/uvr5_lib/separator.py +++ /dev/null @@ -1,252 +0,0 @@ -import os -import sys -import time -import torch -import codecs -import hashlib -import requests -import warnings - -from contextlib import nullcontext -from importlib import import_module - -now_dir = os.getcwd() -sys.path.append(now_dir) - -from main.library.utils import clear_gpu_cache -from main.tools.huggingface import HF_download_file -from main.app.variables import config, translations - -if not config.debug_mode: warnings.filterwarnings("ignore") - -class Separator: - def __init__( - self, - logger, - model_file_dir=config.configs["uvr5_path"], - output_dir=None, - output_format="wav", - output_bitrate=None, - normalization_threshold=0.9, - sample_rate=44100, - mdx_params={ - "hop_length": 1024, - "segment_size": 256, - "overlap": 0.25, - "batch_size": 1, - "enable_denoise": False - }, - demucs_params={ - "segment_size": "Default", - "shifts": 2, - "overlap": 0.25, - "segments_enabled": True - }, - vr_params={ - "batch_size": 1, - "window_size": 512, - "aggression": 5, - "enable_tta": False, - "enable_post_process": False, - "post_process_threshold": 0.2, - "high_end_process": False - } - ): - self.logger = logger - self.logger.info(translations["separator_info"].format(output_dir=output_dir, output_format=output_format)) - - self.model_file_dir = model_file_dir - self.output_dir = output_dir if output_dir is not None else now_dir - - os.makedirs(self.model_file_dir, exist_ok=True) - os.makedirs(self.output_dir, exist_ok=True) - - self.output_format = output_format if output_format is not None else "wav" - self.output_bitrate = output_bitrate - self.normalization_threshold = normalization_threshold - - if normalization_threshold <= 0 or normalization_threshold > 1: raise ValueError - self.sample_rate = int(sample_rate) - - self.arch_specific_params = { - "MDX": mdx_params, - "Demucs": demucs_params, - "VR": vr_params - } - - self.torch_device = None - self.torch_device_cpu = None - self.torch_device_mps = None - self.onnx_execution_provider = None - self.model_instance = None - self.model_type = None - self.setup_torch_device() - - def setup_torch_device(self): - self.torch_device_cpu = torch.device("cpu") - providers = config.providers.copy() - - if len(providers) >= 2 and "CPUExecutionProvider" in providers: providers.remove("CPUExecutionProvider") - self.onnx_execution_provider = providers - - if providers[0] in ["CUDAExecutionProvider", "TensorrtExecutionProvider"]: - self.logger.info(translations["running_in_backends"].format(backends='CUDA')) - self.logger.info(translations["onnx_have"].format(backends='CUDA')) - self.torch_device = torch.device("cuda") - return - elif providers[0] in ["ROCMExecutionProvider", "MIGraphXExecutionProvider"]: - self.logger.info(translations["running_in_backends"].format(backends='HIP' if config.is_zluda else 'ROCM')) - self.logger.info(translations["onnx_have"].format(backends='HIP' if config.is_zluda else 'ROCM')) - self.torch_device = torch.device("cuda") - return - elif providers[0] in ["OpenVINOExecutionProvider", "DnnlExecutionProvider"]: - self.logger.info(translations["running_in_backends"].format(backends='XPU')) - self.logger.info(translations["onnx_have"].format(backends='XPU')) - self.torch_device = torch.device("xpu") - return - elif providers[0] in ["DmlExecutionProvider"]: - backends = (config.device.replace("privateuseone", "dml") if not config.device.startswith("cuda") else config.device.replace("cuda", "hip")).split(":")[0].upper() - self.logger.info(translations["running_in_backends"].format(backends=backends)) - self.logger.info(translations["onnx_have"].format(backends=backends)) - self.torch_device = torch.device(config.device) - return - elif providers[0] in ["CoreMLExecutionProvider"]: - self.logger.info(translations["running_in_backends"].format(backends='MPS')) - self.logger.info(translations["onnx_have"].format(backends='MPS')) - self.torch_device = torch.device("mps") - return - else: - self.logger.info(translations["running_in_cpu"]) - self.logger.warning(translations["onnx_not_have"]) - self.torch_device = self.torch_device_cpu - return - - def download_file_if_not_exists(self, url, output_path): - if os.path.isfile(output_path): return - HF_download_file(url, output_path) - - def list_supported_model_files(self): - response = requests.get( - codecs.decode( - "uggcf://uhttvatsnpr.pb/NauC/Ivrganzrfr-EIP-Cebwrpg/enj/znva/wfba/hie_zbqryf.wfba", - "rot13" - ) - ) - - response.raise_for_status() - model_downloads_list = response.json() - - return { - "MDX": { - **model_downloads_list["mdx_download_list"], - **model_downloads_list["mdx_download_vip_list"] - }, - "Demucs": { - **model_downloads_list["demucs_download_list"] - }, - "VR": { - **model_downloads_list["vr_download_list"] - } - } - - def download_model_files(self, model_filename): - model_repo = codecs.decode("uggcf://uhttvatsnpr.pb/NauC/Ivrganzrfr-EIP-Cebwrpg/erfbyir/znva/hie5_zbqryf", "rot13") - model_path = os.path.join(self.model_file_dir, model_filename) - supported_models = self.list_supported_model_files() - - for model_type, model_list in supported_models.items(): - for _, files in model_list.items(): - if isinstance(files, str) and files == model_filename: - try: - self.download_file_if_not_exists(f"{model_repo}/MDX/{model_filename}", model_path) - except: - try: - self.download_file_if_not_exists(f"{model_repo}/VR/{model_filename}", model_path) - except: - self.download_file_if_not_exists(f"{model_repo}/Demucs/{model_filename}", model_path) - - return model_type, model_path - elif isinstance(files, dict) and any(model_filename in (k, v) for k, v in files.items()): - for _, file_val in files.items(): - self.download_file_if_not_exists(f"{model_repo}/Demucs/{file_val}", os.path.join(self.model_file_dir, file_val)) - - return model_type, model_path - - raise ValueError - - def load_model_data(self, model_path = None, model_type = None): - if model_type == "Demucs": model_data = {} - elif model_path is not None: - try: - with open(model_path, "rb") as f: - f.seek(-10000 * 1024, 2) - - model_hash = hashlib.md5(f.read()).hexdigest() - except IOError: - model_hash = hashlib.md5(open(model_path, "rb").read()).hexdigest() - - response = requests.get(codecs.decode("uggcf://uhttvatsnpr.pb/NauC/Ivrganzrfr-EIP-Cebwrpg/enj/znva/wfba/zbqry_qngn.wfba", "rot13")) - response.raise_for_status() - model_data_object = response.json() - - if model_hash in model_data_object: model_data = model_data_object[model_hash] - else: raise ValueError - else: raise ValueError - - return model_data - - def load_model(self, model_filename): - self.logger.info(translations["loading_model"].format(model_filename=model_filename)) - model_type, model_path = self.download_model_files(model_filename) - - common_params = { - "logger": self.logger, - "torch_device": self.torch_device, - "torch_device_cpu": self.torch_device_cpu, - "torch_device_mps": self.torch_device_mps, - "onnx_execution_provider": self.onnx_execution_provider, - "model_name": model_filename.split(".")[0], - "model_path": model_path, - "model_data": self.load_model_data(model_path, model_type), - "output_format": self.output_format, - "output_bitrate": self.output_bitrate, - "output_dir": self.output_dir, - "normalization_threshold": self.normalization_threshold, - "output_single_stem": None, - "invert_using_spec": False, - "sample_rate": self.sample_rate - } - separator_classes = { - "MDX": "mdx_separator.MDXSeparator", - "Demucs": "demucs_separator.DemucsSeparator", - "VR": "vr_separator.VRSeparator" - } - - if model_type not in self.arch_specific_params or model_type not in separator_classes: - raise ValueError(translations["model_type_not_support"].format(model_type=model_type)) - - module_name, class_name = separator_classes[model_type].split(".") - separator_class = getattr(import_module(f"main.library.uvr5_lib.{module_name}"), class_name) - self.model_instance = separator_class(common_config=common_params, arch_config=self.arch_specific_params[model_type]) - self.model_type = model_type - - def separate(self, audio_file_path): - self.logger.info(f"{translations['starting_separator']}: {audio_file_path}") - separate_start_time = time.perf_counter() - - autocast = torch.amp.autocast( - self.torch_device.type, - enabled=config.is_half, - dtype=torch.float16 if config.is_half else torch.float32 - ) if not self.torch_device.type.startswith(("ocl", "privateuseone")) and self.model_type != "Demucs" else nullcontext() - - with autocast: - output_files = self.model_instance.separate(audio_file_path) - - clear_gpu_cache() - self.model_instance.clear_file_specific_paths() - - self.logger.debug(translations["separator_success_3"]) - self.logger.info(f"{translations['separator_duration']}: {time.strftime('%H:%M:%S', time.gmtime(int(time.perf_counter() - separate_start_time)))}") - - return output_files \ No newline at end of file diff --git a/main/library/uvr5_lib/spec_utils.py b/main/library/uvr5_lib/spec_utils.py deleted file mode 100644 index 3bc508d791f36b70b3338a7e01337d09df3a5460..0000000000000000000000000000000000000000 --- a/main/library/uvr5_lib/spec_utils.py +++ /dev/null @@ -1,1102 +0,0 @@ -import os -import six -import sys -import librosa -import tempfile -import platform -import audioread -import subprocess - -import numpy as np -import soundfile as sf - -from scipy.signal import correlate, hilbert - -sys.path.append(os.getcwd()) - -from main.app.variables import translations, logger - -OPERATING_SYSTEM = platform.system() -SYSTEM_ARCH = platform.platform() -SYSTEM_PROC = platform.processor() -ARM = "arm" -AUTO_PHASE = "Automatic" -POSITIVE_PHASE = "Positive Phase" -NEGATIVE_PHASE = "Negative Phase" -NONE_P = ("None",) -BASE_PATH_RUB = sys._MEIPASS if getattr(sys, 'frozen', False) else os.path.dirname(os.path.abspath(__file__)) -DEVNULL = open(os.devnull, 'w') if six.PY2 else subprocess.DEVNULL -MAX_SPEC = "Max Spec" -MIN_SPEC = "Min Spec" -AVERAGE = "Average" - -is_macos = False -progress_value, last_update_time = 0, 0 - -if OPERATING_SYSTEM == "Darwin": - wav_resolution = "polyphase" if SYSTEM_PROC == ARM or ARM in SYSTEM_ARCH else "sinc_fastest" - wav_resolution_float_resampling = "kaiser_best" if SYSTEM_PROC == ARM or ARM in SYSTEM_ARCH else wav_resolution - is_macos = True -else: - wav_resolution = "sinc_fastest" - wav_resolution_float_resampling = wav_resolution - -def crop_center(h1, h2): - h1_shape = h1.size() - h2_shape = h2.size() - - if h1_shape[3] == h2_shape[3]: return h1 - elif h1_shape[3] < h2_shape[3]: raise ValueError - - s_time = (h1_shape[3] - h2_shape[3]) // 2 - - h1 = h1[:, :, :, s_time:s_time + h2_shape[3]] - return h1 - -def preprocess(X_spec): - return np.abs(X_spec), np.angle(X_spec) - -def make_padding(width, cropsize, offset): - roi_size = cropsize - offset * 2 - - if roi_size == 0: roi_size = cropsize - return offset, roi_size - (width % roi_size) + offset, roi_size - -def normalize(wave, max_peak=1.0): - maxv = np.abs(wave).max() - - if maxv > max_peak: wave *= max_peak / maxv - return wave - -def auto_transpose(audio_array): - if audio_array.shape[1] == 2: return audio_array.T - return audio_array - -def write_array_to_mem(audio_data, subtype): - if isinstance(audio_data, np.ndarray): - import io - - audio_buffer = io.BytesIO() - sf.write(audio_buffer, audio_data, 44100, subtype=subtype, format="WAV") - - audio_buffer.seek(0) - return audio_buffer - else: return audio_data - -def spectrogram_to_image(spec, mode="magnitude"): - if mode == "magnitude": y = np.log10((np.abs(spec) if np.iscomplexobj(spec) else spec)**2 + 1e-8) - elif mode == "phase": y = np.angle(spec) if np.iscomplexobj(spec) else spec - - y -= y.min() - y *= 255 / y.max() - img = np.uint8(y) - - if y.ndim == 3: - img = img.transpose(1, 2, 0) - img = np.concatenate([np.max(img, axis=2, keepdims=True), img], axis=2) - - return img - -def reduce_vocal_aggressively(X, y, softmask): - y_mag_tmp = np.abs(y) - v_mag_tmp = np.abs(X - y) - - return np.clip(y_mag_tmp - v_mag_tmp * (v_mag_tmp > y_mag_tmp) * softmask, 0, np.inf) * np.exp(1.0j * np.angle(y)) - -def merge_artifacts(y_mask, thres=0.01, min_range=64, fade_size=32): - mask = y_mask - - try: - if min_range < fade_size * 2: raise ValueError - - idx = np.where(y_mask.min(axis=(0, 1)) > thres)[0] - start_idx = np.insert(idx[np.where(np.diff(idx) != 1)[0] + 1], 0, idx[0]) - end_idx = np.append(idx[np.where(np.diff(idx) != 1)[0]], idx[-1]) - artifact_idx = np.where(end_idx - start_idx > min_range)[0] - weight = np.zeros_like(y_mask) - - if len(artifact_idx) > 0: - start_idx = start_idx[artifact_idx] - end_idx = end_idx[artifact_idx] - old_e = None - - for s, e in zip(start_idx, end_idx): - if old_e is not None and s - old_e < fade_size: s = old_e - fade_size * 2 - - if s != 0: weight[:, :, s : s + fade_size] = np.linspace(0, 1, fade_size) - else: s -= fade_size - - if e != y_mask.shape[2]: weight[:, :, e - fade_size : e] = np.linspace(1, 0, fade_size) - else: e += fade_size - - weight[:, :, s + fade_size : e - fade_size] = 1 - old_e = e - - v_mask = 1 - y_mask - y_mask += weight * v_mask - mask = y_mask - except Exception as e: - import traceback - logger.error(f'{translations["not_success"]} {type(e).__name__}: {e}\n{traceback.format_exc()}') - - return mask - -def align_wave_head_and_tail(a, b): - l = min([a[0].size, b[0].size]) - return a[:l, :l], b[:l, :l] - -def convert_channels(spec, mp, band): - cc = mp.param["band"][str(band)].get("convert_channels") - - if "mid_side_c" == cc: - spec_left = np.add(spec[0], spec[1] * 0.25) - spec_right = np.subtract(spec[1], spec[0] * 0.25) - elif "mid_side" == cc: - spec_left = np.add(spec[0], spec[1]) / 2 - spec_right = np.subtract(spec[0], spec[1]) - elif "stereo_n" == cc: - spec_left = np.add(spec[0], spec[1] * 0.25) / 0.9375 - spec_right = np.add(spec[1], spec[0] * 0.25) / 0.9375 - else: return spec - - return np.asfortranarray([spec_left, spec_right]) - -def combine_spectrograms(specs, mp, is_v51_model=False): - l = min([specs[i].shape[2] for i in specs]) - spec_c = np.zeros(shape=(2, mp.param["bins"] + 1, l), dtype=np.complex64) - offset = 0 - bands_n = len(mp.param["band"]) - - for d in range(1, bands_n + 1): - h = mp.param["band"][str(d)]["crop_stop"] - mp.param["band"][str(d)]["crop_start"] - spec_c[:, offset : offset + h, :l] = specs[d][:, mp.param["band"][str(d)]["crop_start"] : mp.param["band"][str(d)]["crop_stop"], :l] - offset += h - - if offset > mp.param["bins"]: raise ValueError - - if mp.param["pre_filter_start"] > 0: - if is_v51_model: spec_c *= get_lp_filter_mask(spec_c.shape[1], mp.param["pre_filter_start"], mp.param["pre_filter_stop"]) - else: - if bands_n == 1: spec_c = fft_lp_filter(spec_c, mp.param["pre_filter_start"], mp.param["pre_filter_stop"]) - else: - import math - gp = 1 - - for b in range(mp.param["pre_filter_start"] + 1, mp.param["pre_filter_stop"]): - g = math.pow(10, -(b - mp.param["pre_filter_start"]) * (3.5 - gp) / 20.0) - gp = g - spec_c[:, b, :] *= g - - return np.asfortranarray(spec_c) - -def wave_to_spectrogram(wave, hop_length, n_fft, mp, band, is_v51_model=False): - if wave.ndim == 1: wave = np.asfortranarray([wave, wave]) - - if not is_v51_model: - if mp.param["reverse"]: - wave_left = np.flip(np.asfortranarray(wave[0])) - wave_right = np.flip(np.asfortranarray(wave[1])) - elif mp.param["mid_side"]: - wave_left = np.asfortranarray(np.add(wave[0], wave[1]) / 2) - wave_right = np.asfortranarray(np.subtract(wave[0], wave[1])) - elif mp.param["mid_side_b2"]: - wave_left = np.asfortranarray(np.add(wave[1], wave[0] * 0.5)) - wave_right = np.asfortranarray(np.subtract(wave[0], wave[1] * 0.5)) - else: - wave_left = np.asfortranarray(wave[0]) - wave_right = np.asfortranarray(wave[1]) - else: - wave_left = np.asfortranarray(wave[0]) - wave_right = np.asfortranarray(wave[1]) - - spec_left = librosa.stft(wave_left, n_fft=n_fft, hop_length=hop_length) - spec_right = librosa.stft(wave_right, n_fft=n_fft, hop_length=hop_length) - - spec = np.asfortranarray([spec_left, spec_right]) - - if is_v51_model: spec = convert_channels(spec, mp, band) - return spec - -def spectrogram_to_wave(spec, hop_length=1024, mp={}, band=0, is_v51_model=True): - spec_left = np.asfortranarray(spec[0]) - spec_right = np.asfortranarray(spec[1]) - - wave_left = librosa.istft(spec_left, hop_length=hop_length) - wave_right = librosa.istft(spec_right, hop_length=hop_length) - - if is_v51_model: - cc = mp.param["band"][str(band)].get("convert_channels") - - if "mid_side_c" == cc: - return np.asfortranarray([ - np.subtract(wave_left / 1.0625, wave_right / 4.25), - np.add(wave_right / 1.0625, wave_left / 4.25) - ]) - elif "mid_side" == cc: - return np.asfortranarray([ - np.add(wave_left, wave_right / 2), - np.subtract(wave_left, wave_right / 2) - ]) - elif "stereo_n" == cc: - return np.asfortranarray([ - np.subtract(wave_left, wave_right * 0.25), - np.subtract(wave_right, wave_left * 0.25) - ]) - else: - if mp.param["reverse"]: - return np.asfortranarray([ - np.flip(wave_left), - np.flip(wave_right) - ]) - elif mp.param["mid_side"]: - return np.asfortranarray([ - np.add(wave_left, wave_right / 2), - np.subtract(wave_left, wave_right / 2) - ]) - elif mp.param["mid_side_b2"]: - return np.asfortranarray([ - np.add(wave_right / 1.25, 0.4 * wave_left), - np.subtract(wave_left / 1.25, 0.4 * wave_right) - ]) - - return np.asfortranarray([wave_left, wave_right]) - -def cmb_spectrogram_to_wave(spec_m, mp, extra_bins_h=None, extra_bins=None, is_v51_model=False): - bands_n = len(mp.param["band"]) - offset = 0 - - for d in range(1, bands_n + 1): - bp = mp.param["band"][str(d)] - spec_s = np.zeros(shape=(2, bp["n_fft"] // 2 + 1, spec_m.shape[2]), dtype=complex) - h = bp["crop_stop"] - bp["crop_start"] - spec_s[:, bp["crop_start"] : bp["crop_stop"], :] = spec_m[:, offset : offset + h, :] - offset += h - - if d == bands_n: - if extra_bins_h: - max_bin = bp["n_fft"] // 2 - spec_s[:, max_bin - extra_bins_h : max_bin, :] = extra_bins[:, :extra_bins_h, :] - - if bp["hpf_start"] > 0: - if is_v51_model: spec_s *= get_hp_filter_mask(spec_s.shape[1], bp["hpf_start"], bp["hpf_stop"] - 1) - else: spec_s = fft_hp_filter(spec_s, bp["hpf_start"], bp["hpf_stop"] - 1) - - wave = ( - spectrogram_to_wave( - spec_s, - bp["hl"], - mp, - d, - is_v51_model - ) - ) if bands_n == 1 else ( - np.add( - wave, - spectrogram_to_wave( - spec_s, - bp["hl"], - mp, - d, - is_v51_model - ) - ) - ) - else: - sr = mp.param["band"][str(d + 1)]["sr"] - - if d == 1: - if is_v51_model: - spec_s *= get_lp_filter_mask( - spec_s.shape[1], - bp["lpf_start"], - bp["lpf_stop"] - ) - else: - spec_s = fft_lp_filter( - spec_s, - bp["lpf_start"], - bp["lpf_stop"] - ) - - try: - wave = librosa.resample( - spectrogram_to_wave( - spec_s, - bp["hl"], - mp, - d, - is_v51_model - ), - orig_sr=bp["sr"], - target_sr=sr, - res_type="soxr_vhq" - ) - except ValueError as e: - logger.error(f"{translations['resample_error']}: {e}") - logger.error(f"{translations['shapes']} Spec_s: {spec_s.shape}, SR: {sr}, {translations['wav_resolution']}: {wav_resolution}") - else: - if is_v51_model: - spec_s *= get_hp_filter_mask( - spec_s.shape[1], - bp["hpf_start"], - bp["hpf_stop"] - 1 - ) - - spec_s *= get_lp_filter_mask( - spec_s.shape[1], - bp["lpf_start"], - bp["lpf_stop"] - ) - else: - spec_s = fft_hp_filter( - spec_s, - bp["hpf_start"], - bp["hpf_stop"] - 1 - ) - - spec_s = fft_lp_filter( - spec_s, - bp["lpf_start"], - bp["lpf_stop"] - ) - - try: - wave = librosa.resample( - np.add(wave, spectrogram_to_wave(spec_s, bp["hl"], mp, d, is_v51_model)), - orig_sr=bp["sr"], - target_sr=sr, - res_type="soxr_vhq" - ) - except ValueError as e: - logger.error(f"{translations['resample_error']}: {e}") - logger.error(f"{translations['shapes']} Spec_s: {spec_s.shape}, SR: {sr}, {translations['wav_resolution']}: {wav_resolution}") - - return wave - -def get_lp_filter_mask(n_bins, bin_start, bin_stop): - return np.concatenate([ - np.ones((bin_start - 1, 1)), - np.linspace(1, 0, bin_stop - bin_start + 1)[:, None], - np.zeros((n_bins - bin_stop, 1)) - ], axis=0) - -def get_hp_filter_mask(n_bins, bin_start, bin_stop): - return np.concatenate([ - np.zeros((bin_stop + 1, 1)), - np.linspace(0, 1, 1 + bin_start - bin_stop)[:, None], - np.ones((n_bins - bin_start - 2, 1)) - ], axis=0) - -def fft_lp_filter(spec, bin_start, bin_stop): - g = 1.0 - - for b in range(bin_start, bin_stop): - g -= 1 / (bin_stop - bin_start) - spec[:, b, :] = g * spec[:, b, :] - - spec[:, bin_stop:, :] *= 0 - return spec - -def fft_hp_filter(spec, bin_start, bin_stop): - g = 1.0 - - for b in range(bin_start, bin_stop, -1): - g -= 1 / (bin_start - bin_stop) - spec[:, b, :] = g * spec[:, b, :] - - spec[:, 0 : bin_stop + 1, :] *= 0 - return spec - -def spectrogram_to_wave_old(spec, hop_length=1024): - if spec.ndim == 2: - wave = librosa.istft( - spec, - hop_length=hop_length - ) - elif spec.ndim == 3: - wave = np.asfortranarray([ - librosa.istft( - np.asfortranarray(spec[0]), - hop_length=hop_length - ), - librosa.istft( - np.asfortranarray(spec[1]), - hop_length=hop_length - ) - ]) - - return wave - -def wave_to_spectrogram_old(wave, hop_length, n_fft): - return np.asfortranarray([ - librosa.stft( - np.asfortranarray(wave[0]), - n_fft=n_fft, - hop_length=hop_length - ), - librosa.stft( - np.asfortranarray(wave[1]), - n_fft=n_fft, - hop_length=hop_length - ) - ]) - -def mirroring(a, spec_m, input_high_end, mp): - if "mirroring" == a: - mirror = np.flip( - np.abs(spec_m[:, mp.param["pre_filter_start"] - 10 - input_high_end.shape[1] : mp.param["pre_filter_start"] - 10, :]), 1 - ) * np.exp(1.0j * np.angle(input_high_end)) - - return np.where(np.abs(input_high_end) <= np.abs(mirror), input_high_end, mirror) - - if "mirroring2" == a: - mi = np.multiply( - np.flip( - np.abs(spec_m[:, mp.param["pre_filter_start"] - 10 - input_high_end.shape[1] : mp.param["pre_filter_start"] - 10, :]), 1 - ), input_high_end * 1.7 - ) - - return np.where(np.abs(input_high_end) <= np.abs(mi), input_high_end, mi) - -def adjust_aggr(mask, is_non_accom_stem, aggressiveness): - aggr = aggressiveness["value"] * 2 - - if aggr != 0: - if is_non_accom_stem: - aggr = 1 - aggr - - if np.any(aggr > 10) or np.any(aggr < -10): logger.warning(f"{translations['warnings']}: {aggr}") - - aggr = [aggr, aggr] - - if aggressiveness["aggr_correction"] is not None: - aggr[0] += aggressiveness["aggr_correction"]["left"] - aggr[1] += aggressiveness["aggr_correction"]["right"] - - for ch in range(2): - mask[ch, : aggressiveness["split_bin"]] = np.power(mask[ch, : aggressiveness["split_bin"]], 1 + aggr[ch] / 3) - mask[ch, aggressiveness["split_bin"] :] = np.power(mask[ch, aggressiveness["split_bin"] :], 1 + aggr[ch]) - - return mask - -def stft(wave, nfft, hl): - return np.asfortranarray([ - librosa.stft( - np.asfortranarray(wave[0]), - n_fft=nfft, - hop_length=hl - ), - librosa.stft( - np.asfortranarray(wave[1]), - n_fft=nfft, - hop_length=hl - ) - ]) - -def istft(spec, hl): - return np.asfortranarray([ - librosa.istft( - np.asfortranarray(spec[0]), - hop_length=hl - ), - librosa.istft( - np.asfortranarray(spec[1]), - hop_length=hl - ) - ]) - -def spec_effects(wave, algorithm="Default", value=None): - if np.isnan(wave).any() or np.isinf(wave).any(): logger.warning(f"{translations['warnings_2']}: {wave.shape}") - spec = [stft(wave[0], 2048, 1024), stft(wave[1], 2048, 1024)] - - if algorithm == "Min_Mag": wave = istft(np.where(np.abs(spec[1]) <= np.abs(spec[0]), spec[1], spec[0]), 1024) - elif algorithm == "Max_Mag": wave = istft(np.where(np.abs(spec[1]) >= np.abs(spec[0]), spec[1], spec[0]), 1024) - elif algorithm == "Default": wave = (wave[1] * value) + (wave[0] * (1 - value)) - elif algorithm == "Invert_p": - X_mag, y_mag = np.abs(spec[0]), np.abs(spec[1]) - wave = istft(spec[1] - np.where(X_mag >= y_mag, X_mag, y_mag) * np.exp(1.0j * np.angle(spec[0])), 1024) - - return wave - -def spectrogram_to_wave_no_mp(spec, n_fft=2048, hop_length=1024): - wave = librosa.istft(spec, n_fft=n_fft, hop_length=hop_length) - if wave.ndim == 1: wave = np.asfortranarray([wave, wave]) - - return wave - -def wave_to_spectrogram_no_mp(wave): - spec = librosa.stft(wave, n_fft=2048, hop_length=1024) - - if spec.ndim == 1: spec = np.asfortranarray([spec, spec]) - return spec - -def invert_audio(specs, invert_p=True): - ln = min([specs[0].shape[2], specs[1].shape[2]]) - specs[0] = specs[0][:, :, :ln] - specs[1] = specs[1][:, :, :ln] - - if invert_p: - X_mag, y_mag = np.abs(specs[0]), np.abs(specs[1]) - v_spec = specs[1] - np.where(X_mag >= y_mag, X_mag, y_mag) * np.exp(1.0j * np.angle(specs[0])) - else: - specs[1] = reduce_vocal_aggressively(specs[0], specs[1], 0.2) - v_spec = specs[0] - specs[1] - - return v_spec - -def invert_stem(mixture, stem): - return -spectrogram_to_wave_no_mp(invert_audio([wave_to_spectrogram_no_mp(mixture), wave_to_spectrogram_no_mp(stem)])).T - -def ensembling(a, inputs, is_wavs=False): - for i in range(1, len(inputs)): - if i == 1: input = inputs[0] - - if is_wavs: - ln = min([input.shape[1], inputs[i].shape[1]]) - input = input[:, :ln] - inputs[i] = inputs[i][:, :ln] - else: - ln = min([input.shape[2], inputs[i].shape[2]]) - input = input[:, :, :ln] - inputs[i] = inputs[i][:, :, :ln] - - if MIN_SPEC == a: input = np.where(np.abs(inputs[i]) <= np.abs(input), inputs[i], input) - if MAX_SPEC == a: input = np.where(np.abs(inputs[i]) >= np.abs(input), inputs[i], input) - - return input - -def ensemble_for_align(waves): - specs = [] - - for wav in waves: - spec = wave_to_spectrogram_no_mp(wav.T) - specs.append(spec) - - wav_aligned = spectrogram_to_wave_no_mp(ensembling(MIN_SPEC, specs)).T - wav_aligned = match_array_shapes(wav_aligned, waves[1], is_swap=True) - - return wav_aligned - -def ensemble_inputs(audio_input, algorithm, is_normalization, wav_type_set, save_path, is_wave=False, is_array=False): - wavs_ = [] - - if algorithm == AVERAGE: - output = average_audio(audio_input) - samplerate = 44100 - else: - specs = [] - - for i in range(len(audio_input)): - wave, samplerate = librosa.load(audio_input[i], mono=False, sr=44100) - wavs_.append(wave) - specs.append( wave if is_wave else wave_to_spectrogram_no_mp(wave)) - - wave_shapes = [w.shape[1] for w in wavs_] - target_shape = wavs_[wave_shapes.index(max(wave_shapes))] - - output = ensembling(algorithm, specs, is_wavs=True) if is_wave else spectrogram_to_wave_no_mp(ensembling(algorithm, specs)) - output = to_shape(output, target_shape.shape) - - sf.write(save_path, normalize(output.T, is_normalization), samplerate, subtype=wav_type_set) - -def to_shape(x, target_shape): - padding_list = [] - - for x_dim, target_dim in zip(x.shape, target_shape): - padding_list.append((0, target_dim - x_dim)) - - return np.pad(x, tuple(padding_list), mode="constant") - -def to_shape_minimize(x, target_shape): - padding_list = [] - - for x_dim, target_dim in zip(x.shape, target_shape): - padding_list.append((0, target_dim - x_dim)) - - return np.pad(x, tuple(padding_list), mode="constant") - -def detect_leading_silence(audio, sr, silence_threshold=0.007, frame_length=1024): - if len(audio.shape) == 2: - channel = np.argmax(np.sum(np.abs(audio), axis=1)) - audio = audio[channel] - - for i in range(0, len(audio), frame_length): - if np.max(np.abs(audio[i : i + frame_length])) > silence_threshold: return (i / sr) * 1000 - - return (len(audio) / sr) * 1000 - -def adjust_leading_silence(target_audio, reference_audio, silence_threshold=0.01, frame_length=1024): - def find_silence_end(audio): - if len(audio.shape) == 2: - channel = np.argmax(np.sum(np.abs(audio), axis=1)) - audio_mono = audio[channel] - else: audio_mono = audio - - for i in range(0, len(audio_mono), frame_length): - if np.max(np.abs(audio_mono[i : i + frame_length])) > silence_threshold: return i - - return len(audio_mono) - - ref_silence_end = find_silence_end(reference_audio) - target_silence_end = find_silence_end(target_audio) - silence_difference = ref_silence_end - target_silence_end - - try: - silence_difference_p = ((ref_silence_end / 44100) * 1000) - ((target_silence_end / 44100) * 1000) - except Exception as e: - pass - - if silence_difference > 0: return np.hstack((np.zeros((target_audio.shape[0], silence_difference))if len(target_audio.shape) == 2 else np.zeros(silence_difference), target_audio)) - elif silence_difference < 0: return target_audio[:, -silence_difference:]if len(target_audio.shape) == 2 else target_audio[-silence_difference:] - else: return target_audio - -def match_array_shapes(array_1, array_2, is_swap=False): - - if is_swap: array_1, array_2 = array_1.T, array_2.T - - if array_1.shape[1] > array_2.shape[1]: array_1 = array_1[:, : array_2.shape[1]] - elif array_1.shape[1] < array_2.shape[1]: - padding = array_2.shape[1] - array_1.shape[1] - array_1 = np.pad(array_1, ((0, 0), (0, padding)), "constant", constant_values=0) - - if is_swap: array_1, array_2 = array_1.T, array_2.T - - return array_1 - -def match_mono_array_shapes(array_1, array_2): - if len(array_1) > len(array_2): array_1 = array_1[: len(array_2)] - elif len(array_1) < len(array_2): - padding = len(array_2) - len(array_1) - array_1 = np.pad(array_1, (0, padding), "constant", constant_values=0) - - return array_1 - -def change_pitch_semitones(y, sr, semitone_shift): - factor = 2 ** (semitone_shift / 12) - y_pitch_tuned = [] - - for y_channel in y: - y_pitch_tuned.append(librosa.resample(y_channel, orig_sr=sr, target_sr=sr * factor, res_type="soxr_vhq")) - - y_pitch_tuned = np.array(y_pitch_tuned) - new_sr = sr * factor - - return y_pitch_tuned, new_sr - -def augment_audio(export_path, audio_file, rate, is_normalization, wav_type_set, save_format=None, is_pitch=False, is_time_correction=True): - wav, sr = librosa.load(audio_file, sr=44100, mono=False) - if wav.ndim == 1: wav = np.asfortranarray([wav, wav]) - - if not is_time_correction: wav_mix = change_pitch_semitones(wav, 44100, semitone_shift=-rate)[0] - else: - if is_pitch: wav_1, wav_2 = pitch_shift(wav[0], sr, rate, rbargs=None), pitch_shift(wav[1], sr, rate, rbargs=None) - else: wav_1, wav_2 = time_stretch(wav[0], sr, rate, rbargs=None), time_stretch(wav[1], sr, rate, rbargs=None) - - if wav_1.shape > wav_2.shape: wav_2 = to_shape(wav_2, wav_1.shape) - if wav_1.shape < wav_2.shape: wav_1 = to_shape(wav_1, wav_2.shape) - - wav_mix = np.asfortranarray([wav_1, wav_2]) - - sf.write(export_path, normalize(wav_mix.T, is_normalization), sr, subtype=wav_type_set) - save_format(export_path) - - -def average_audio(audio): - waves, wave_shapes, final_waves = [], [], [] - - for i in range(len(audio)): - wave = librosa.load(audio[i], sr=44100, mono=False) - waves.append(wave[0]) - wave_shapes.append(wave[0].shape[1]) - - wave_shapes_index = wave_shapes.index(max(wave_shapes)) - target_shape = waves[wave_shapes_index] - - waves.pop(wave_shapes_index) - final_waves.append(target_shape) - - for n_array in waves: - wav_target = to_shape(n_array, target_shape.shape) - final_waves.append(wav_target) - - waves = sum(final_waves) - return waves / len(audio) - -def average_dual_sources(wav_1, wav_2, value): - if wav_1.shape > wav_2.shape: wav_2 = to_shape(wav_2, wav_1.shape) - if wav_1.shape < wav_2.shape: wav_1 = to_shape(wav_1, wav_2.shape) - - return (wav_1 * value) + (wav_2 * (1 - value)) - -def reshape_sources(wav_1, wav_2): - if wav_1.shape > wav_2.shape: wav_2 = to_shape(wav_2, wav_1.shape) - - if wav_1.shape < wav_2.shape: - ln = min([wav_1.shape[1], wav_2.shape[1]]) - wav_2 = wav_2[:, :ln] - - ln = min([wav_1.shape[1], wav_2.shape[1]]) - wav_1 = wav_1[:, :ln] - wav_2 = wav_2[:, :ln] - - return wav_2 - -def reshape_sources_ref(wav_1_shape, wav_2): - if wav_1_shape > wav_2.shape: wav_2 = to_shape(wav_2, wav_1_shape) - return wav_2 - -def combine_arrarys(audio_sources, is_swap=False): - source = np.zeros_like(max(audio_sources, key=np.size)) - - for v in audio_sources: - v = match_array_shapes(v, source, is_swap=is_swap) - source += v - - return source - -def combine_audio(paths, audio_file_base=None, wav_type_set="FLOAT", save_format=None): - source = combine_arrarys([load_audio(i) for i in paths]) - save_path = f"{audio_file_base}_combined.wav" - sf.write(save_path, source.T, 44100, subtype=wav_type_set) - save_format(save_path) - -def reduce_mix_bv(inst_source, voc_source, reduction_rate=0.9): - return combine_arrarys([inst_source * (1 - reduction_rate), voc_source], is_swap=True) - -def organize_inputs(inputs): - input_list = {"target": None, "reference": None, "reverb": None, "inst": None} - - for i in inputs: - if i.endswith("_(Vocals).wav"): input_list["reference"] = i - elif "_RVC_" in i: input_list["target"] = i - elif i.endswith("reverbed_stem.wav"): input_list["reverb"] = i - elif i.endswith("_(Instrumental).wav"): input_list["inst"] = i - - return input_list - -def check_if_phase_inverted(wav1, wav2, is_mono=False): - if not is_mono: - wav1 = np.mean(wav1, axis=0) - wav2 = np.mean(wav2, axis=0) - - return np.corrcoef(wav1[:1000], wav2[:1000])[0, 1] < 0 - -def rerun_mp3(audio_file): - with audioread.audio_open(audio_file) as f: - track_length = int(f.duration) - - return track_length - -def align_audio( - file1, - file2, - file2_aligned, - file_subtracted, - wav_type_set, - is_save_aligned, - command_Text, - save_format, - align_window, - align_intro_val, - db_analysis, - set_progress_bar, - phase_option, - phase_shifts, - is_match_silence, - is_spec_match -): - global progress_value - progress_value = 0 - is_mono = False - - def get_diff(a, b): - return np.correlate(a, b, "full").argmax() - (b.shape[0] - 1) - - def progress_bar(length): - global progress_value - progress_value += 1 - - if (0.90 / length * progress_value) >= 0.9: length = progress_value + 1 - set_progress_bar(0.1, (0.9 / length * progress_value)) - - if file1.endswith(".mp3") and is_macos: - length1 = rerun_mp3(file1) - wav1, sr1 = librosa.load(file1, duration=length1, sr=44100, mono=False) - else: - wav1, sr1 = librosa.load(file1, sr=44100, mono=False) - - if file2.endswith(".mp3") and is_macos: - length2 = rerun_mp3(file2) - wav2, sr2 = librosa.load(file2, duration=length2, sr=44100, mono=False) - else: - wav2, sr2 = librosa.load(file2, sr=44100, mono=False) - - if wav1.ndim == 1 and wav2.ndim == 1: is_mono = True - elif wav1.ndim == 1: wav1 = np.asfortranarray([wav1, wav1]) - elif wav2.ndim == 1: wav2 = np.asfortranarray([wav2, wav2]) - - if phase_option == AUTO_PHASE: - if check_if_phase_inverted(wav1, wav2, is_mono=is_mono): wav2 = -wav2 - elif phase_option == POSITIVE_PHASE: wav2 = +wav2 - elif phase_option == NEGATIVE_PHASE: wav2 = -wav2 - - if is_match_silence: wav2 = adjust_leading_silence(wav2, wav1) - - wav1_length = int(librosa.get_duration(y=wav1, sr=44100)) - wav2_length = int(librosa.get_duration(y=wav2, sr=44100)) - - if not is_mono: - wav1 = wav1.transpose() - wav2 = wav2.transpose() - - wav2_org = wav2.copy() - - command_Text(translations["process_file"]) - seconds_length = min(wav1_length, wav2_length) - wav2_aligned_sources = [] - - for sec_len in align_intro_val: - sec_seg = 1 if sec_len == 1 else int(seconds_length // sec_len) - index = sr1 * sec_seg - - if is_mono: - samp1, samp2 = wav1[index : index + sr1], wav2[index : index + sr1] - diff = get_diff(samp1, samp2) - else: - index = sr1 * sec_seg - samp1, samp2 = wav1[index : index + sr1, 0], wav2[index : index + sr1, 0] - samp1_r, samp2_r = wav1[index : index + sr1, 1], wav2[index : index + sr1, 1] - diff, _ = get_diff(samp1, samp2), get_diff(samp1_r, samp2_r) - - if diff > 0: wav2_aligned = np.append(np.zeros(diff) if is_mono else np.zeros((diff, 2)), wav2_org, axis=0) - elif diff < 0: wav2_aligned = wav2_org[-diff:] - else: wav2_aligned = wav2_org - - if not any(np.array_equal(wav2_aligned, source) for source in wav2_aligned_sources): wav2_aligned_sources.append(wav2_aligned) - - unique_sources = len(wav2_aligned_sources) - sub_mapper_big_mapper = {} - - for s in wav2_aligned_sources: - wav2_aligned = match_mono_array_shapes(s, wav1) if is_mono else match_array_shapes(s, wav1, is_swap=True) - - if align_window: - wav_sub = time_correction( - wav1, - wav2_aligned, - seconds_length, - align_window=align_window, - db_analysis=db_analysis, - progress_bar=progress_bar, - unique_sources=unique_sources, - phase_shifts=phase_shifts - ) - - sub_mapper_big_mapper = {**sub_mapper_big_mapper, **{np.abs(wav_sub).mean(): wav_sub}} - else: - wav2_aligned = wav2_aligned * np.power(10, db_analysis[0] / 20) - - for db_adjustment in db_analysis[1]: - sub_mapper_big_mapper = {**sub_mapper_big_mapper, **{np.abs(wav_sub).mean(): wav1 - (wav2_aligned * (10 ** (db_adjustment / 20)))}} - - wav_sub = ensemble_for_align( - list(sub_mapper_big_mapper.values()) - ) if is_spec_match and len(list(sub_mapper_big_mapper.values())) >= 2 else ensemble_wav( - list(sub_mapper_big_mapper.values()) - ) - - wav_sub = np.clip(wav_sub, -1, +1) - command_Text(translations["save_instruments"]) - - if is_save_aligned or is_spec_match: - wav1 = match_mono_array_shapes(wav1, wav_sub) if is_mono else match_array_shapes(wav1, wav_sub, is_swap=True) - wav2_aligned = wav1 - wav_sub - - if is_spec_match: - if wav1.ndim == 1 and wav2.ndim == 1: - wav2_aligned = np.asfortranarray([wav2_aligned, wav2_aligned]).T - wav1 = np.asfortranarray([wav1, wav1]).T - - wav2_aligned = ensemble_for_align([wav2_aligned, wav1]) - wav_sub = wav1 - wav2_aligned - - if is_save_aligned: - sf.write(file2_aligned, wav2_aligned, sr1, subtype=wav_type_set) - save_format(file2_aligned) - - sf.write(file_subtracted, wav_sub, sr1, subtype=wav_type_set) - save_format(file_subtracted) - -def phase_shift_hilbert(signal, degree): - analytic_signal = hilbert(signal) - return np.cos(np.radians(degree)) * analytic_signal.real - np.sin(np.radians(degree)) * analytic_signal.imag - -def get_phase_shifted_tracks(track, phase_shift): - if phase_shift == 180: return [track, -track] - - step = phase_shift - end = 180 - (180 % step) if 180 % step == 0 else 181 - phase_range = range(step, end, step) - flipped_list = [track, -track] - - for i in phase_range: - flipped_list.extend([phase_shift_hilbert(track, i), phase_shift_hilbert(track, -i)]) - - return flipped_list - -def time_correction( - mix, - instrumental, - seconds_length, - align_window, - db_analysis, - sr=44100, - progress_bar=None, - unique_sources=None, - phase_shifts=NONE_P -): - def align_tracks(track1, track2): - shifted_tracks = {} - track2 = track2 * np.power(10, db_analysis[0] / 20) - track2_flipped = [track2] if phase_shifts == 190 else get_phase_shifted_tracks(track2, phase_shifts) - - for db_adjustment in db_analysis[1]: - for t in track2_flipped: - track2_adjusted = t * (10 ** (db_adjustment / 20)) - track2_shifted = np.roll(track2_adjusted, shift=np.argmax(np.abs(correlate(track1, track2_adjusted))) - (len(track1) - 1)) - shifted_tracks[np.abs(track1 - track2_shifted).mean()] = track2_shifted - - return shifted_tracks[min(shifted_tracks.keys())] - - assert mix.shape == instrumental.shape, translations["assert"].format(mixshape=mix.shape, instrumentalshape=instrumental.shape) - seconds_length = seconds_length // 2 - - sub_mapper = {} - progress_update_interval, total_iterations = 120, 0 - - if len(align_window) > 2: progress_update_interval = 320 - - for secs in align_window: - step = secs / 2 - window_size = int(sr * secs) - step_size = int(sr * step) - - if len(mix.shape) == 1: total_iterations += ((len(range(0, len(mix) - window_size, step_size)) // progress_update_interval) * unique_sources) - else: total_iterations += ((len(range(0, len(mix[:, 0]) - window_size, step_size)) * 2 // progress_update_interval) * unique_sources) - - for secs in align_window: - sub = np.zeros_like(mix) - divider = np.zeros_like(mix) - window_size = int(sr * secs) - step_size = int(sr * secs / 2) - window = np.hanning(window_size) - - if len(mix.shape) == 1: - counter = 0 - - for i in range(0, len(mix) - window_size, step_size): - counter += 1 - if counter % progress_update_interval == 0: progress_bar(total_iterations) - - window_mix = mix[i : i + window_size] * window - window_instrumental = instrumental[i : i + window_size] * window - window_instrumental_aligned = align_tracks(window_mix, window_instrumental) - sub[i : i + window_size] += window_mix - window_instrumental_aligned - divider[i : i + window_size] += window - else: - counter = 0 - - for ch in range(mix.shape[1]): - for i in range(0, len(mix[:, ch]) - window_size, step_size): - counter += 1 - - if counter % progress_update_interval == 0: progress_bar(total_iterations) - - window_mix = mix[i : i + window_size, ch] * window - window_instrumental = instrumental[i : i + window_size, ch] * window - window_instrumental_aligned = align_tracks(window_mix, window_instrumental) - sub[i : i + window_size, ch] += window_mix - window_instrumental_aligned - divider[i : i + window_size, ch] += window - - return ensemble_wav(list({**sub_mapper, **{np.abs(sub).mean(): np.where(divider > 1e-6, sub / divider, sub)}}.values()), split_size=12) - -def ensemble_wav(waveforms, split_size=240): - waveform_thirds = {i: np.array_split(waveform, split_size) for i, waveform in enumerate(waveforms)} - final_waveform = [] - for third_idx in range(split_size): - final_waveform.append(waveform_thirds[np.argmin([np.abs(waveform_thirds[i][third_idx]).mean() for i in range(len(waveforms))])][third_idx]) - - return np.concatenate(final_waveform) - -def ensemble_wav_min(waveforms): - for i in range(1, len(waveforms)): - if i == 1: wave = waveforms[0] - ln = min(len(wave), len(waveforms[i])) - wave = wave[:ln] - waveforms[i] = waveforms[i][:ln] - wave = np.where(np.abs(waveforms[i]) <= np.abs(wave), waveforms[i], wave) - - return wave - -def align_audio_test(wav1, wav2, sr1=44100): - def get_diff(a, b): - return np.correlate(a, b, "full").argmax() - (b.shape[0] - 1) - - wav1 = wav1.transpose() - wav2 = wav2.transpose() - wav2_org = wav2.copy() - index = sr1 - diff = get_diff(wav1[index : index + sr1, 0], wav2[index : index + sr1, 0]) - - if diff > 0: wav2_aligned = np.append(np.zeros((diff, 1)), wav2_org, axis=0) - elif diff < 0: wav2_aligned = wav2_org[-diff:] - else: wav2_aligned = wav2_org - return wav2_aligned - -def load_audio(audio_file): - wav, _ = librosa.load(audio_file, sr=44100, mono=False) - if wav.ndim == 1: wav = np.asfortranarray([wav, wav]) - return wav - -def __rubberband(y, sr, **kwargs): - assert sr > 0 - fd, infile = tempfile.mkstemp(suffix='.wav') - os.close(fd) - fd, outfile = tempfile.mkstemp(suffix='.wav') - os.close(fd) - - sf.write(infile, y, sr) - - try: - arguments = [os.path.join(BASE_PATH_RUB, 'rubberband'), '-q'] - for key, value in six.iteritems(kwargs): - arguments.append(str(key)) - arguments.append(str(value)) - - arguments.extend([infile, outfile]) - subprocess.check_call(arguments, stdout=DEVNULL, stderr=DEVNULL) - - y_out, _ = sf.read(outfile, always_2d=True) - if y.ndim == 1: y_out = np.squeeze(y_out) - except OSError as e: - six.raise_from(RuntimeError(translations["rubberband"]), e) - finally: - os.unlink(infile) - os.unlink(outfile) - - return y_out - -def time_stretch(y, sr, rate, rbargs=None): - if rate <= 0: raise ValueError(translations["rate"]) - if rate == 1.0: return y - if rbargs is None: rbargs = dict() - - rbargs.setdefault('--tempo', rate) - return __rubberband(y, sr, **rbargs) - -def pitch_shift(y, sr, n_steps, rbargs=None): - if n_steps == 0: return y - if rbargs is None: rbargs = dict() - - rbargs.setdefault('--pitch', n_steps) - return __rubberband(y, sr, **rbargs) \ No newline at end of file diff --git a/main/library/uvr5_lib/vr_network/layers.py b/main/library/uvr5_lib/vr_network/layers.py deleted file mode 100644 index 597cb59d51f9d4cc81d1b496374495d3c8cd953e..0000000000000000000000000000000000000000 --- a/main/library/uvr5_lib/vr_network/layers.py +++ /dev/null @@ -1,265 +0,0 @@ -import os -import sys -import torch - -import torch.nn as nn -import torch.nn.functional as F - -sys.path.append(os.getcwd()) - -from main.library.uvr5_lib import spec_utils - -class Conv2DBNActiv(nn.Module): - def __init__( - self, - nin, - nout, - ksize=3, - stride=1, - pad=1, - dilation=1, - activ=nn.ReLU - ): - super(Conv2DBNActiv, self).__init__() - self.conv = nn.Sequential( - nn.Conv2d( - nin, - nout, - kernel_size=ksize, - stride=stride, - padding=pad, - dilation=dilation, - bias=False - ), - nn.BatchNorm2d(nout), - activ() - ) - - def __call__(self, input_tensor): - return self.conv(input_tensor) - -class SeperableConv2DBNActiv(nn.Module): - def __init__( - self, - nin, - nout, - ksize=3, - stride=1, - pad=1, - dilation=1, - activ=nn.ReLU - ): - super(SeperableConv2DBNActiv, self).__init__() - self.conv = nn.Sequential( - nn.Conv2d( - nin, - nin, - kernel_size=ksize, - stride=stride, - padding=pad, - dilation=dilation, - groups=nin, - bias=False, - ), - nn.Conv2d( - nin, - nout, - kernel_size=1, - bias=False, - ), - nn.BatchNorm2d(nout), - activ(), - ) - - def __call__(self, input_tensor): - return self.conv(input_tensor) - -class Encoder(nn.Module): - def __init__( - self, - nin, - nout, - ksize=3, - stride=1, - pad=1, - activ=nn.LeakyReLU - ): - super(Encoder, self).__init__() - self.conv1 = Conv2DBNActiv( - nin, - nout, - ksize, - 1, - pad, - activ=activ - ) - self.conv2 = Conv2DBNActiv( - nout, - nout, - ksize, - stride, - pad, - activ=activ - ) - - def __call__(self, input_tensor): - skip = self.conv1(input_tensor) - hidden = self.conv2(skip) - - return hidden, skip - -class Decoder(nn.Module): - def __init__( - self, - nin, - nout, - ksize=3, - stride=1, - pad=1, - activ=nn.ReLU, - dropout=False - ): - super(Decoder, self).__init__() - self.dropout = nn.Dropout2d(0.1) if dropout else None - self.conv = Conv2DBNActiv( - nin, - nout, - ksize, - 1, - pad, - activ=activ - ) - - def __call__(self, input_tensor, skip=None): - input_tensor = F.interpolate( - input_tensor, - scale_factor=2, - mode="bilinear", - align_corners=True - ) - - if skip is not None: - skip = spec_utils.crop_center(skip, input_tensor) - input_tensor = torch.cat([input_tensor, skip], dim=1) - - output_tensor = self.conv(input_tensor) - if self.dropout is not None: - output_tensor = self.dropout(output_tensor) - - return output_tensor - -class ASPPModule(nn.Module): - def __init__(self, nn_architecture, nin, nout, dilations=(4, 8, 16), activ=nn.ReLU): - super(ASPPModule, self).__init__() - self.nn_architecture = nn_architecture - self.six_layer = [129605] - self.seven_layer = [537238, 537227, 33966] - - extra_conv = SeperableConv2DBNActiv( - nin, - nin, - 3, - 1, - dilations[2], - dilations[2], - activ=activ - ) - - self.conv1 = nn.Sequential( - nn.AdaptiveAvgPool2d((1, None)), - Conv2DBNActiv( - nin, - nin, - 1, - 1, - 0, - activ=activ - ) - ) - self.conv2 = Conv2DBNActiv( - nin, - nin, - 1, - 1, - 0, - activ=activ - ) - self.conv3 = SeperableConv2DBNActiv( - nin, - nin, - 3, - 1, - dilations[0], - dilations[0], - activ=activ - ) - self.conv4 = SeperableConv2DBNActiv( - nin, - nin, - 3, - 1, - dilations[1], - dilations[1], - activ=activ - ) - self.conv5 = SeperableConv2DBNActiv( - nin, - nin, - 3, - 1, - dilations[2], - dilations[2], - activ=activ - ) - - if self.nn_architecture in self.six_layer: - self.conv6 = extra_conv - nin_x = 6 - elif self.nn_architecture in self.seven_layer: - self.conv6 = extra_conv - self.conv7 = extra_conv - nin_x = 7 - else: - nin_x = 5 - - self.bottleneck = nn.Sequential( - Conv2DBNActiv( - nin * nin_x, - nout, - 1, - 1, - 0, - activ=activ - ), - nn.Dropout2d(0.1) - ) - - def forward(self, input_tensor): - _, _, h, w = input_tensor.size() - - feat1 = F.interpolate( - self.conv1(input_tensor), - size=(h, w), - mode="bilinear", - align_corners=True - ) - - feat2 = self.conv2(input_tensor) - feat3 = self.conv3(input_tensor) - feat4 = self.conv4(input_tensor) - feat5 = self.conv5(input_tensor) - - if self.nn_architecture in self.six_layer: - feat6 = self.conv6(input_tensor) - - out = torch.cat((feat1, feat2, feat3, feat4, feat5, feat6), dim=1) - elif self.nn_architecture in self.seven_layer: - feat6 = self.conv6(input_tensor) - feat7 = self.conv7(input_tensor) - - out = torch.cat((feat1, feat2, feat3, feat4, feat5, feat6, feat7), dim=1) - else: - out = torch.cat((feat1, feat2, feat3, feat4, feat5), dim=1) - - bottleneck_output = self.bottleneck(out) - return bottleneck_output \ No newline at end of file diff --git a/main/library/uvr5_lib/vr_network/layers_new.py b/main/library/uvr5_lib/vr_network/layers_new.py deleted file mode 100644 index d3c2e4f3edcb26ce09dede185931d245f2eafb61..0000000000000000000000000000000000000000 --- a/main/library/uvr5_lib/vr_network/layers_new.py +++ /dev/null @@ -1,245 +0,0 @@ -import os -import sys -import torch - -import torch.nn as nn -import torch.nn.functional as F - -sys.path.append(os.getcwd()) - -from main.library.uvr5_lib import spec_utils - -class Conv2DBNActiv(nn.Module): - def __init__( - self, - nin, - nout, - ksize=3, - stride=1, - pad=1, - dilation=1, - activ=nn.ReLU - ): - super(Conv2DBNActiv, self).__init__() - self.conv = nn.Sequential( - nn.Conv2d( - nin, - nout, - kernel_size=ksize, - stride=stride, - padding=pad, - dilation=dilation, - bias=False - ), - nn.BatchNorm2d(nout), - activ() - ) - - def __call__(self, input_tensor): - return self.conv(input_tensor) - -class Encoder(nn.Module): - def __init__( - self, - nin, - nout, - ksize=3, - stride=1, - pad=1, - activ=nn.LeakyReLU - ): - super(Encoder, self).__init__() - self.conv1 = Conv2DBNActiv( - nin, - nout, - ksize, - stride, - pad, - activ=activ - ) - - self.conv2 = Conv2DBNActiv( - nout, - nout, - ksize, - 1, - pad, - activ=activ - ) - - def __call__(self, input_tensor): - hidden = self.conv1(input_tensor) - hidden = self.conv2(hidden) - - return hidden - -class Decoder(nn.Module): - def __init__( - self, - nin, - nout, - ksize=3, - stride=1, - pad=1, - activ=nn.ReLU, - dropout=False - ): - super(Decoder, self).__init__() - self.dropout = nn.Dropout2d(0.1) if dropout else None - self.conv1 = Conv2DBNActiv( - nin, - nout, - ksize, - 1, - pad, - activ=activ - ) - - def __call__(self, input_tensor, skip=None): - input_tensor = F.interpolate( - input_tensor, - scale_factor=2, - mode="bilinear", - align_corners=True - ) - - if skip is not None: - skip = spec_utils.crop_center(skip, input_tensor) - input_tensor = torch.cat([input_tensor, skip], dim=1) - - hidden = self.conv1(input_tensor) - - if self.dropout is not None: - hidden = self.dropout(hidden) - - return hidden - -class ASPPModule(nn.Module): - def __init__( - self, - nin, - nout, - dilations=(4, 8, 12), - activ=nn.ReLU, - dropout=False - ): - super(ASPPModule, self).__init__() - self.conv1 = nn.Sequential( - nn.AdaptiveAvgPool2d((1, None)), - Conv2DBNActiv( - nin, - nout, - 1, - 1, - 0, - activ=activ - ) - ) - - self.conv2 = Conv2DBNActiv( - nin, - nout, - 1, - 1, - 0, - activ=activ - ) - - self.conv3 = Conv2DBNActiv( - nin, - nout, - 3, - 1, - dilations[0], - dilations[0], - activ=activ - ) - - self.conv4 = Conv2DBNActiv( - nin, - nout, - 3, - 1, - dilations[1], - dilations[1], - activ=activ - ) - - self.conv5 = Conv2DBNActiv( - nin, - nout, - 3, - 1, - dilations[2], - dilations[2], - activ=activ - ) - - self.bottleneck = Conv2DBNActiv( - nout * 5, - nout, - 1, - 1, - 0, - activ=activ - ) - - self.dropout = nn.Dropout2d(0.1) if dropout else None - - def forward(self, input_tensor): - _, _, h, w = input_tensor.size() - - out = self.bottleneck( - torch.cat(( - F.interpolate( - self.conv1(input_tensor), - size=(h, w), - mode="bilinear", - align_corners=True - ), - self.conv2(input_tensor), - self.conv3(input_tensor), - self.conv4(input_tensor), - self.conv5(input_tensor) - ), dim=1)) - - if self.dropout is not None: - out = self.dropout(out) - - return out - -class LSTMModule(nn.Module): - def __init__( - self, - nin_conv, - nin_lstm, - nout_lstm - ): - super(LSTMModule, self).__init__() - self.conv = Conv2DBNActiv( - nin_conv, - 1, - 1, - 1, - 0 - ) - - self.lstm = nn.LSTM( - input_size=nin_lstm, - hidden_size=nout_lstm // 2, - bidirectional=True - ) - - self.dense = nn.Sequential( - nn.Linear(nout_lstm, nin_lstm), - nn.BatchNorm1d(nin_lstm), - nn.ReLU() - ) - - def forward(self, input_tensor): - N, _, nbins, nframes = input_tensor.size() - - hidden, _ = self.lstm(self.conv(input_tensor)[:, 0].permute(2, 0, 1)) - hidden = self.dense(hidden.reshape(-1, hidden.size()[-1])).reshape(nframes, N, 1, nbins) - - return hidden.permute(1, 2, 3, 0) \ No newline at end of file diff --git a/main/library/uvr5_lib/vr_network/model_param_init.py b/main/library/uvr5_lib/vr_network/model_param_init.py deleted file mode 100644 index 1ad0c1fc28b9f2df13c8f9c7d6d7bb23ebfc7ab5..0000000000000000000000000000000000000000 --- a/main/library/uvr5_lib/vr_network/model_param_init.py +++ /dev/null @@ -1,43 +0,0 @@ -import json -import pickle - -default_param = {} -default_param["bins"] = -1 -default_param["unstable_bins"] = -1 -default_param["stable_bins"] = -1 -default_param["sr"] = 44100 -default_param["pre_filter_start"] = -1 -default_param["pre_filter_stop"] = -1 -default_param["band"] = {} - -N_BINS = "n_bins" - -def int_keys(pairs): - result_dict = {} - - for key, value in pairs: - if isinstance(key, str) and key.isdigit(): key = int(key) - result_dict[key] = value - - return result_dict - -class ModelParameters(object): - def __init__( - self, - config_path="", - key_in_bin=None - ): - if config_path.endswith(".bin"): - with open(config_path, "rb") as f: - data = pickle.load(f) - self.param = data[key_in_bin] - else: - with open(config_path, "r", encoding="utf-8") as f: - self.param = json.loads(f.read(), object_pairs_hook=int_keys) - - for k in ["mid_side", "mid_side_b", "mid_side_b2", "stereo_w", "stereo_n", "reverse"]: - if k not in self.param: - self.param[k] = False - - if N_BINS in self.param: - self.param["bins"] = self.param[N_BINS] \ No newline at end of file diff --git a/main/library/uvr5_lib/vr_network/nets.py b/main/library/uvr5_lib/vr_network/nets.py deleted file mode 100644 index b5313b65c2895a77401a5998948e50c0158b7157..0000000000000000000000000000000000000000 --- a/main/library/uvr5_lib/vr_network/nets.py +++ /dev/null @@ -1,280 +0,0 @@ -import os -import sys -import torch - -import torch.nn as nn -import torch.nn.functional as F - -sys.path.append(os.getcwd()) - -from main.library.uvr5_lib.vr_network import layers - -class BaseASPPNet(nn.Module): - def __init__( - self, - nn_architecture, - nin, - ch, - dilations=(4, 8, 16) - ): - super(BaseASPPNet, self).__init__() - self.nn_architecture = nn_architecture - - self.enc1 = layers.Encoder( - nin, - ch, - 3, - 2, - 1 - ) - self.enc2 = layers.Encoder( - ch, - ch * 2, - 3, - 2, - 1 - ) - self.enc3 = layers.Encoder( - ch * 2, - ch * 4, - 3, - 2, - 1 - ) - self.enc4 = layers.Encoder( - ch * 4, - ch * 8, - 3, - 2, - 1 - ) - - if self.nn_architecture == 129605: - self.enc5 = layers.Encoder( - ch * 8, - ch * 16, - 3, - 2, - 1 - ) - self.aspp = layers.ASPPModule( - nn_architecture, - ch * 16, - ch * 32, - dilations - ) - self.dec5 = layers.Decoder( - ch * (16 + 32), - ch * 16, - 3, - 1, - 1 - ) - else: - self.aspp = layers.ASPPModule( - nn_architecture, - ch * 8, - ch * 16, - dilations - ) - - self.dec4 = layers.Decoder( - ch * (8 + 16), - ch * 8, - 3, - 1, - 1 - ) - self.dec3 = layers.Decoder( - ch * (4 + 8), - ch * 4, - 3, - 1, - 1 - ) - self.dec2 = layers.Decoder( - ch * (2 + 4), - ch * 2, - 3, - 1, - 1 - ) - self.dec1 = layers.Decoder( - ch * (1 + 2), - ch, - 3, - 1, - 1 - ) - - def __call__(self, input_tensor): - hidden_state, encoder_output1 = self.enc1(input_tensor) - hidden_state, encoder_output2 = self.enc2(hidden_state) - hidden_state, encoder_output3 = self.enc3(hidden_state) - hidden_state, encoder_output4 = self.enc4(hidden_state) - - if self.nn_architecture == 129605: - hidden_state, encoder_output5 = self.enc5(hidden_state) - hidden_state = self.dec5(self.aspp(hidden_state), encoder_output5) - else: - hidden_state = self.aspp(hidden_state) - - hidden_state = self.dec1( - self.dec2( - self.dec3(self.dec4(hidden_state, encoder_output4), encoder_output3), - encoder_output2 - ), - encoder_output1 - ) - return hidden_state - -def determine_model_capacity(n_fft_bins, nn_architecture): - sp_model_arch = [31191, 33966, 129605] - hp_model_arch = [123821, 123812] - hp2_model_arch = [537238, 537227] - - if nn_architecture in sp_model_arch: - model_capacity_data = [ - (2, 16), - (2, 16), - (18, 8, 1, 1, 0), - (8, 16), - (34, 16, 1, 1, 0), - (16, 32), - (32, 2, 1), - (16, 2, 1), - (16, 2, 1) - ] - - if nn_architecture in hp_model_arch: - model_capacity_data = [ - (2, 32), - (2, 32), - (34, 16, 1, 1, 0), - (16, 32), - (66, 32, 1, 1, 0), - (32, 64), - (64, 2, 1), - (32, 2, 1), - (32, 2, 1) - ] - - if nn_architecture in hp2_model_arch: - model_capacity_data = [ - (2, 64), - (2, 64), - (66, 32, 1, 1, 0), - (32, 64), - (130, 64, 1, 1, 0), - (64, 128), - (128, 2, 1), - (64, 2, 1), - (64, 2, 1) - ] - - cascaded = CascadedASPPNet - model = cascaded( - n_fft_bins, - model_capacity_data, - nn_architecture - ) - - return model - -class CascadedASPPNet(nn.Module): - def __init__( - self, - n_fft, - model_capacity_data, - nn_architecture - ): - super(CascadedASPPNet, self).__init__() - self.stg1_low_band_net = BaseASPPNet( - nn_architecture, - *model_capacity_data[0] - ) - self.stg1_high_band_net = BaseASPPNet( - nn_architecture, - *model_capacity_data[1] - ) - - self.stg2_bridge = layers.Conv2DBNActiv(*model_capacity_data[2]) - self.stg2_full_band_net = BaseASPPNet( - nn_architecture, - *model_capacity_data[3] - ) - - self.stg3_bridge = layers.Conv2DBNActiv(*model_capacity_data[4]) - self.stg3_full_band_net = BaseASPPNet( - nn_architecture, - *model_capacity_data[5] - ) - - self.out = nn.Conv2d( - *model_capacity_data[6], - bias=False - ) - self.aux1_out = nn.Conv2d( - *model_capacity_data[7], - bias=False - ) - self.aux2_out = nn.Conv2d( - *model_capacity_data[8], - bias=False - ) - - self.max_bin = n_fft // 2 - self.output_bin = n_fft // 2 + 1 - self.offset = 128 - - def forward(self, input_tensor): - mix = input_tensor.detach() - - input_tensor = input_tensor.clone() - input_tensor = input_tensor[:, :, : self.max_bin] - - bandwidth = input_tensor.size()[2] // 2 - aux1 = torch.cat([ - self.stg1_low_band_net(input_tensor[:, :, :bandwidth]), - self.stg1_high_band_net(input_tensor[:, :, bandwidth:]) - ], dim=2) - - hidden_state = torch.cat([input_tensor, aux1], dim=1) - aux2 = self.stg2_full_band_net(self.stg2_bridge(hidden_state)) - - hidden_state = torch.cat([input_tensor, aux1, aux2], dim=1) - hidden_state = self.stg3_full_band_net(self.stg3_bridge(hidden_state)) - - mask = self.out(hidden_state).sigmoid() - mask = F.pad( - input=mask, - pad=(0, 0, 0, self.output_bin - mask.size()[2]), - mode="replicate" - ) - - if self.training: - aux1 = self.aux1_out(aux1).sigmoid() - aux1 = F.pad( - input=aux1, - pad=(0, 0, 0, self.output_bin - aux1.size()[2]), - mode="replicate" - ) - - aux2 = self.aux2_out(aux2).sigmoid() - aux2 = F.pad( - input=aux2, - pad=(0, 0, 0, self.output_bin - aux2.size()[2]), - mode="replicate" - ) - - return mask * mix, aux1 * mix, aux2 * mix - else: - return mask - - def predict_mask(self, input_tensor): - mask = self.forward(input_tensor) - - if self.offset > 0: - mask = mask[:, :, :, self.offset : -self.offset] - - return mask \ No newline at end of file diff --git a/main/library/uvr5_lib/vr_network/nets_new.py b/main/library/uvr5_lib/vr_network/nets_new.py deleted file mode 100644 index adbfff3328d66d135bb95dc02062cc61c5bb547d..0000000000000000000000000000000000000000 --- a/main/library/uvr5_lib/vr_network/nets_new.py +++ /dev/null @@ -1,252 +0,0 @@ -import os -import sys -import torch - -import torch.nn as nn -import torch.nn.functional as F - -sys.path.append(os.getcwd()) - -from main.library.uvr5_lib.vr_network import layers_new as layers - -class BaseNet(nn.Module): - def __init__( - self, - nin, - nout, - nin_lstm, - nout_lstm, - dilations=((4, 2), (8, 4), (12, 6)) - ): - super(BaseNet, self).__init__() - self.enc1 = layers.Conv2DBNActiv( - nin, - nout, - 3, - 1, - 1 - ) - self.enc2 = layers.Encoder( - nout, - nout * 2, - 3, - 2, - 1 - ) - self.enc3 = layers.Encoder( - nout * 2, - nout * 4, - 3, - 2, - 1 - ) - self.enc4 = layers.Encoder( - nout * 4, - nout * 6, - 3, - 2, - 1 - ) - self.enc5 = layers.Encoder( - nout * 6, - nout * 8, - 3, - 2, - 1 - ) - self.aspp = layers.ASPPModule( - nout * 8, - nout * 8, - dilations, - dropout=True - ) - self.dec4 = layers.Decoder( - nout * (6 + 8), - nout * 6, - 3, - 1, - 1 - ) - self.dec3 = layers.Decoder( - nout * (4 + 6), - nout * 4, - 3, - 1, - 1 - ) - self.dec2 = layers.Decoder( - nout * (2 + 4), - nout * 2, - 3, - 1, - 1 - ) - self.lstm_dec2 = layers.LSTMModule( - nout * 2, - nin_lstm, - nout_lstm - ) - self.dec1 = layers.Decoder( - nout * (1 + 2) + 1, - nout * 1, - 3, - 1, - 1 - ) - - def __call__(self, input_tensor): - encoded1 = self.enc1(input_tensor) - encoded2 = self.enc2(encoded1) - encoded3 = self.enc3(encoded2) - encoded4 = self.enc4(encoded3) - - bottleneck = self.dec2( - self.dec3( - self.dec4( - self.aspp(self.enc5(encoded4)), - encoded4 - ), - encoded3 - ), - encoded2 - ) - - bottleneck = self.dec1( - torch.cat([ - bottleneck, - self.lstm_dec2(bottleneck) - ], dim=1), - encoded1 - ) - - return bottleneck - -class CascadedNet(nn.Module): - def __init__( - self, - n_fft, - nn_arch_size=51000, - nout=32, - nout_lstm=128 - ): - super(CascadedNet, self).__init__() - self.max_bin = n_fft // 2 - self.output_bin = n_fft // 2 + 1 - self.nin_lstm = self.max_bin // 2 - self.offset = 64 - nout = 64 if nn_arch_size == 218409 else nout - - self.stg1_low_band_net = nn.Sequential( - BaseNet( - 2, - nout // 2, - self.nin_lstm // 2, - nout_lstm - ), - layers.Conv2DBNActiv( - nout // 2, - nout // 4, - 1, - 1, - 0 - ) - ) - - self.stg1_high_band_net = BaseNet( - 2, - nout // 4, - self.nin_lstm // 2, - nout_lstm // 2 - ) - - self.stg2_low_band_net = nn.Sequential( - BaseNet( - nout // 4 + 2, - nout, - self.nin_lstm // 2, nout_lstm - ), - layers.Conv2DBNActiv( - nout, - nout // 2, - 1, - 1, - 0 - ) - ) - - self.stg2_high_band_net = BaseNet( - nout // 4 + 2, - nout // 2, - self.nin_lstm // 2, - nout_lstm // 2 - ) - self.stg3_full_band_net = BaseNet( - 3 * nout // 4 + 2, - nout, - self.nin_lstm, - nout_lstm - ) - - self.out = nn.Conv2d( - nout, - 2, - 1, - bias=False - ) - self.aux_out = nn.Conv2d( - 3 * nout // 4, - 2, - 1, - bias=False - ) - - def forward(self, input_tensor): - input_tensor = input_tensor[:, :, : self.max_bin] - bandw = input_tensor.size()[2] // 2 - - l1_in = input_tensor[:, :, :bandw] - h1_in = input_tensor[:, :, bandw:] - - l1 = self.stg1_low_band_net(l1_in) - h1 = self.stg1_high_band_net(h1_in) - aux1 = torch.cat([l1, h1], dim=2) - - l2_in = torch.cat([l1_in, l1], dim=1) - h2_in = torch.cat([h1_in, h1], dim=1) - - l2 = self.stg2_low_band_net(l2_in) - h2 = self.stg2_high_band_net(h2_in) - - aux2 = torch.cat([l2, h2], dim=2) - f3_in = torch.cat([input_tensor, aux1, aux2], dim=1) - f3 = self.stg3_full_band_net(f3_in) - - mask = self.out(f3).sigmoid() - mask = F.pad(input=mask, pad=(0, 0, 0, self.output_bin - mask.size()[2]), mode="replicate") - - if self.training: - aux = self.aux_out(torch.cat([aux1, aux2], dim=1)).sigmoid() - aux = F.pad(input=aux, pad=(0, 0, 0, self.output_bin - aux.size()[2]), mode="replicate") - - return mask, aux - else: - return mask - - def predict_mask(self, input_tensor): - mask = self.forward(input_tensor) - - if self.offset > 0: - mask = mask[:, :, :, self.offset : -self.offset] - assert mask.size()[3] > 0 - - return mask - - def predict(self, input_tensor): - mask = self.forward(input_tensor) - pred_mag = input_tensor * mask - - if self.offset > 0: - pred_mag = pred_mag[:, :, :, self.offset : -self.offset] - assert pred_mag.size()[3] > 0 - - return pred_mag \ No newline at end of file diff --git a/main/library/uvr5_lib/vr_separator.py b/main/library/uvr5_lib/vr_separator.py deleted file mode 100644 index f865fa8c69d5f4cb4397dff25556927e20cc2f61..0000000000000000000000000000000000000000 --- a/main/library/uvr5_lib/vr_separator.py +++ /dev/null @@ -1,232 +0,0 @@ -import os -import sys -import math -import torch -import librosa -import audioread - -import numpy as np - -from tqdm import tqdm - -sys.path.append(os.getcwd()) - -from main.app.variables import configs -from main.library.uvr5_lib import spec_utils -from main.library.uvr5_lib.vr_network import nets -from main.library.uvr5_lib.vr_network import nets_new -from main.library.uvr5_lib.common_separator import CommonSeparator -from main.library.uvr5_lib.vr_network.model_param_init import ModelParameters - -class VRSeparator(CommonSeparator): - def __init__(self, common_config, arch_config): - super().__init__(config=common_config) - self.model_capacity = 32, 128 - self.is_vr_51_model = False - - if "nout" in self.model_data.keys() and "nout_lstm" in self.model_data.keys(): - self.model_capacity = self.model_data["nout"], self.model_data["nout_lstm"] - self.is_vr_51_model = True - - self.model_params = ModelParameters(os.path.join(configs["binary_path"], "vr_params.bin"), f"{self.model_data['vr_model_param']}.json") - self.enable_tta = arch_config.get("enable_tta", False) - self.enable_post_process = arch_config.get("enable_post_process", False) - self.post_process_threshold = arch_config.get("post_process_threshold", 0.2) - self.batch_size = arch_config.get("batch_size", 1) - self.window_size = arch_config.get("window_size", 512) - self.high_end_process = arch_config.get("high_end_process", False) - self.input_high_end_h = None - self.input_high_end = None - self.aggression = float(int(arch_config.get("aggression", 5)) / 100) - self.aggressiveness = { - "value": self.aggression, - "split_bin": self.model_params.param["band"]["1"]["crop_stop"], - "aggr_correction": self.model_params.param.get("aggr_correction") - } - self.model_samplerate = self.model_params.param["sr"] - self.wav_subtype = "PCM_16" - - def separate(self, audio_file_path): - self.primary_source = None - self.secondary_source = None - self.audio_file_path = audio_file_path - self.audio_file_base = os.path.splitext(os.path.basename(audio_file_path))[0] - nn_arch_sizes = [31191, 33966, 56817, 123821, 123812, 129605, 218409, 537238, 537227] - vr_5_1_models = [56817, 218409] - model_size = math.ceil(os.stat(self.model_path).st_size / 1024) - nn_arch_size = min(nn_arch_sizes, key=lambda x: abs(x - model_size)) - - if nn_arch_size in vr_5_1_models or self.is_vr_51_model: - self.model_run = nets_new.CascadedNet( - self.model_params.param["bins"] * 2, - nn_arch_size, - nout=self.model_capacity[0], - nout_lstm=self.model_capacity[1] - ) - self.is_vr_51_model = True - else: - self.model_run = nets.determine_model_capacity( - self.model_params.param["bins"] * 2, - nn_arch_size - ) - - self.model_run.load_state_dict(torch.load(self.model_path, map_location="cpu", weights_only=True)) - self.model_run.to(self.torch_device) - - y_spec, v_spec = self.inference_vr(self.loading_mix(), self.torch_device, self.aggressiveness) - y_spec = np.nan_to_num(y_spec, nan=0.0, posinf=0.0, neginf=0.0) - v_spec = np.nan_to_num(v_spec, nan=0.0, posinf=0.0, neginf=0.0) - - output_files = [] - if self.output_single_stem and (self.output_single_stem.lower() != self.primary_stem_name.lower() and self.output_single_stem.lower() != self.secondary_stem_name.lower()): - self.output_single_stem = None - - if not self.output_single_stem or self.output_single_stem.lower() == self.primary_stem_name.lower(): - if not isinstance(self.primary_source, np.ndarray): - self.primary_source = self.spec_to_wav(y_spec).T - if not self.model_samplerate == 44100: - self.primary_source = librosa.resample(self.primary_source.T, orig_sr=self.model_samplerate, target_sr=44100).T - - self.primary_stem_output_path = os.path.join(f"{self.audio_file_base}_({self.primary_stem_name})_{self.model_name}.{self.output_format.lower()}") - self.final_process(self.primary_stem_output_path, self.primary_source, self.primary_stem_name) - output_files.append(self.primary_stem_output_path) - - if not self.output_single_stem or self.output_single_stem.lower() == self.secondary_stem_name.lower(): - if not isinstance(self.secondary_source, np.ndarray): - self.secondary_source = self.spec_to_wav(v_spec).T - if not self.model_samplerate == 44100: - self.secondary_source = librosa.resample(self.secondary_source.T, orig_sr=self.model_samplerate, target_sr=44100).T - - self.secondary_stem_output_path = os.path.join(f"{self.audio_file_base}_({self.secondary_stem_name})_{self.model_name}.{self.output_format.lower()}") - self.final_process(self.secondary_stem_output_path, self.secondary_source, self.secondary_stem_name) - output_files.append(self.secondary_stem_output_path) - - return output_files - - def loading_mix(self): - X_wave, X_spec_s = {}, {} - bands_n = len(self.model_params.param["band"]) - - audio_file = spec_utils.write_array_to_mem(self.audio_file_path, subtype=self.wav_subtype) - is_mp3 = audio_file.endswith(".mp3") if isinstance(audio_file, str) else False - - for d in tqdm(range(bands_n, 0, -1)): - bp = self.model_params.param["band"][str(d)] - - if d == bands_n: - X_wave[d], _ = librosa.load(audio_file, sr=bp["sr"], mono=False, dtype=np.float32, res_type="soxr_vhq") - X_spec_s[d] = spec_utils.wave_to_spectrogram(X_wave[d], bp["hl"], bp["n_fft"], self.model_params, band=d, is_v51_model=self.is_vr_51_model) - - if not np.any(X_wave[d]) and is_mp3: - X_wave[d] = rerun_mp3(audio_file, bp["sr"]) - - if X_wave[d].ndim == 1: - X_wave[d] = np.asarray([X_wave[d], X_wave[d]]) - else: - X_wave[d] = librosa.resample(X_wave[d + 1], orig_sr=self.model_params.param["band"][str(d + 1)]["sr"], target_sr=bp["sr"], res_type="soxr_vhq") - X_spec_s[d] = spec_utils.wave_to_spectrogram(X_wave[d], bp["hl"], bp["n_fft"], self.model_params, band=d, is_v51_model=self.is_vr_51_model) - - if d == bands_n and self.high_end_process: - self.input_high_end_h = (bp["n_fft"] // 2 - bp["crop_stop"]) + (self.model_params.param["pre_filter_stop"] - self.model_params.param["pre_filter_start"]) - self.input_high_end = X_spec_s[d][:, bp["n_fft"] // 2 - self.input_high_end_h : bp["n_fft"] // 2, :] - - X_spec = spec_utils.combine_spectrograms(X_spec_s, self.model_params, is_v51_model=self.is_vr_51_model) - del X_wave, X_spec_s, audio_file - - return X_spec - - def inference_vr(self, X_spec, device, aggressiveness): - def _execute(X_mag_pad, roi_size): - X_dataset = [] - patches = (X_mag_pad.shape[2] - 2 * self.model_run.offset) // roi_size - - for i in tqdm(range(patches)): - start = i * roi_size - X_mag_window = X_mag_pad[:, :, start : start + self.window_size] - X_dataset.append(X_mag_window) - - X_dataset = np.asarray(X_dataset) - self.model_run.eval() - - with torch.no_grad(): - mask = [] - - for i in tqdm(range(0, patches, self.batch_size)): - X_batch = X_dataset[i : i + self.batch_size] - X_batch = torch.from_numpy(X_batch).to(device) - pred = self.model_run.predict_mask(X_batch) - - if not pred.size()[3] > 0: - raise ValueError - - pred = pred.detach().cpu().numpy() - pred = np.concatenate(pred, axis=2) - mask.append(pred) - - if len(mask) == 0: - raise ValueError - - mask = np.concatenate(mask, axis=2) - - return mask - - def postprocess(mask, X_mag, X_phase): - is_non_accom_stem = False - for stem in CommonSeparator.NON_ACCOM_STEMS: - if stem == self.primary_stem_name: is_non_accom_stem = True - - mask = spec_utils.adjust_aggr(mask, is_non_accom_stem, aggressiveness) - if self.enable_post_process: mask = spec_utils.merge_artifacts(mask, thres=self.post_process_threshold) - - y_spec = mask * X_mag * np.exp(1.0j * X_phase) - v_spec = (1 - mask) * X_mag * np.exp(1.0j * X_phase) - - return y_spec, v_spec - - X_mag, X_phase = spec_utils.preprocess(X_spec) - n_frame = X_mag.shape[2] - pad_l, pad_r, roi_size = spec_utils.make_padding(n_frame, self.window_size, self.model_run.offset) - X_mag_pad = np.pad(X_mag, ((0, 0), (0, 0), (pad_l, pad_r)), mode="constant") - X_mag_pad /= X_mag_pad.max() - mask = _execute(X_mag_pad, roi_size) - - if self.enable_tta: - pad_l += roi_size // 2 - pad_r += roi_size // 2 - X_mag_pad = np.pad(X_mag, ((0, 0), (0, 0), (pad_l, pad_r)), mode="constant") - X_mag_pad /= X_mag_pad.max() - mask_tta = _execute(X_mag_pad, roi_size) - mask_tta = mask_tta[:, :, roi_size // 2 :] - mask = (mask[:, :, :n_frame] + mask_tta[:, :, :n_frame]) * 0.5 - else: - mask = mask[:, :, :n_frame] - - y_spec, v_spec = postprocess(mask, X_mag, X_phase) - - return y_spec, v_spec - - def spec_to_wav(self, spec): - if self.high_end_process and isinstance(self.input_high_end, np.ndarray) and self.input_high_end_h: - input_high_end_ = spec_utils.mirroring("mirroring", spec, self.input_high_end, self.model_params) - - wav = spec_utils.cmb_spectrogram_to_wave( - spec, - self.model_params, - self.input_high_end_h, - input_high_end_, - is_v51_model=self.is_vr_51_model - ) - else: - wav = spec_utils.cmb_spectrogram_to_wave( - spec, - self.model_params, - is_v51_model=self.is_vr_51_model - ) - - return wav - -def rerun_mp3(audio_file, sample_rate=44100): - with audioread.audio_open(audio_file) as f: - track_length = int(f.duration) - - return librosa.load(audio_file, duration=track_length, mono=False, sr=sample_rate)[0] \ No newline at end of file diff --git a/main/tools/analyzer.py b/main/tools/analyzer.py deleted file mode 100644 index 8c16be134c8a8ae1362128e53330c1c8679f0b27..0000000000000000000000000000000000000000 --- a/main/tools/analyzer.py +++ /dev/null @@ -1,70 +0,0 @@ -import os -import sys -import librosa -import librosa.display - -import numpy as np -import matplotlib.pyplot as plt - -sys.path.append(os.getcwd()) - -from main.app.variables import translations - -def calculate_features(y, sr): - stft = np.abs(librosa.stft(y)) - return stft, librosa.get_duration(y=y, sr=sr), librosa.feature.spectral_centroid(S=stft, sr=sr)[0], librosa.feature.spectral_bandwidth(S=stft, sr=sr)[0], librosa.feature.spectral_rolloff(S=stft, sr=sr)[0] - -def plot_title(title): - plt.suptitle(title, fontsize=16, fontweight="bold") - -def plot_spectrogram(sr, stft, duration, cmap="inferno"): - plt.subplot(3, 1, 1) - plt.imshow(librosa.amplitude_to_db(stft, ref=np.max), origin="lower", extent=[0, duration, 0, sr / 1000], aspect="auto", cmap=cmap) - plt.colorbar(format="%+2.0f dB") - plt.xlabel(translations["times"]) - plt.ylabel(translations["frequencykhz"]) - plt.title(translations["spectrogram"]) - -def plot_waveform(y, sr): - plt.subplot(3, 1, 2) - librosa.display.waveshow(y, sr=sr) - plt.xlabel(translations["times"]) - plt.ylabel(translations["amplitude"]) - plt.title(translations["waveform"]) - -def plot_features(times, cent, bw, rolloff): - plt.subplot(3, 1, 3) - plt.plot(times, cent, label=translations["centroid"], color="b") - plt.plot(times, bw, label=translations["bandwidth"], color="g") - plt.plot(times, rolloff, label=translations["rolloff"], color="r") - plt.xlabel(translations["times"]) - plt.title(translations["features"]) - plt.legend() - -def analyze_audio(audio_file): - save_plot_path = "assets/audio_analysis.png" - - y, sr = librosa.load(audio_file, sr=None) - stft, duration, cent, bw, rolloff = calculate_features(y, sr) - - plt.figure(figsize=(12, 10)) - - plot_title(translations["analysis"]) - plot_spectrogram(sr, stft, duration) - plot_waveform(y, sr) - plot_features(librosa.times_like(cent), cent, bw, rolloff) - - plt.tight_layout() - - if save_plot_path: plt.savefig(save_plot_path, bbox_inches="tight", dpi=300) - plt.close() - - audio_info = translations["analysis_info"].format( - sr=sr, - duration=(str(round(duration, 2)) + translations["seconds"] if duration < 60 else str(round(duration / 60, 2)) + translations["minutes"]), - length=len(y), - bit=librosa.get_samplerate(audio_file), - channel=translations["mono"] if y.ndim == 1 else translations["stereo"] - ) - - return audio_info, save_plot_path \ No newline at end of file diff --git a/main/tools/export_svc.py b/main/tools/export_svc.py deleted file mode 100644 index f6fb1fa2a2de31e66373bf4e4db93ff356a2f72d..0000000000000000000000000000000000000000 --- a/main/tools/export_svc.py +++ /dev/null @@ -1,78 +0,0 @@ -import os -import sys -import json -import torch -import hashlib -import datetime - -sys.path.append(os.getcwd()) - -from main.app.variables import translations - -def svc_converter(model_path, config_path, output_path, is_half=False): - try: - model_dict, weight = {}, {} - model = torch.load(model_path, map_location="cpu", weights_only=True) - - if "model" not in model: - return translations["not_svc"] - - for key, value in model["model"].items(): - if torch.is_tensor(value): - value = value.half() if is_half else value.float() - - if key.startswith("enc_p.enc_."): weight[key.replace("enc_p.enc_.", "enc_p.encoder.")] = value - elif key.startswith("f0_decoder."): continue - else: weight[key] = value - - config = json.load(open(config_path, "r", encoding="utf-8")) - - model_dict["weight"] = weight - model_dict["config"] = [ - config["data"]["filter_length"] // 2 + 1, - config["train"]["segment_size"] / config["data"]["hop_length"], - config["model"]["inter_channels"], - config["model"]["hidden_channels"], - config["model"]["filter_channels"], - config["model"]["n_heads"], - config["model"]["n_layers"], - config["model"]["kernel_size"], - config["model"]["p_dropout"], - config["model"]["resblock"], - config["model"]["resblock_kernel_sizes"], - config["model"]["resblock_dilation_sizes"], - config["model"]["upsample_rates"], - config["model"]["upsample_initial_channel"], - config["model"]["upsample_kernel_sizes"], - config["model"]["n_speakers"], - config["model"]["gin_channels"], - config["data"]["sampling_rate"] - ] - - epoch = model.get("iteration", 0) - - try: - step = ( - int(os.path.basename(model_path).replace("G_", "").replace(".pth", "")) - ) if model_path.startswith("G_") and model_path.endswith(".pth") else epoch - except: - step = epoch - - model_dict["epoch"] = f"{epoch}epoch" - model_dict["step"] = step - model_dict["sr"] = config["data"]["sampling_rate"] - model_dict["f0"] = True - model_dict["version"] = "v2" if config["model"]["ssl_dim"] == 768 else "v1" - model_dict["creation_date"] = datetime.datetime.now().isoformat() - model_dict["model_hash"] = hashlib.sha256(f"{str(weight)} {epoch} {step} {datetime.datetime.now().isoformat()}".encode()).hexdigest() - model_dict["model_name"] = os.path.basename(output_path).replace(".pth", "") - model_dict["author"] = "" - model_dict["vocoder"] = "Default" - model_dict["energy"] = False - model_dict["speakers_id"] = config["model"]["n_speakers"] - model_dict["architecture"] = "SVC" - - torch.save(model_dict, output_path) - return output_path - except: - return translations["not_svc"] \ No newline at end of file diff --git a/main/tools/gdown.py b/main/tools/gdown.py deleted file mode 100644 index 2bdf24731be3622d860186ffbb67107eabab7c24..0000000000000000000000000000000000000000 --- a/main/tools/gdown.py +++ /dev/null @@ -1,175 +0,0 @@ -import os -import re -import sys -import json -import tqdm -import codecs -import tempfile -import requests - -from urllib.parse import urlparse, parse_qs, unquote - -sys.path.append(os.getcwd()) - -from main.app.variables import translations - -def parse_url(url): - parsed = urlparse(url) - is_download_link = parsed.path.endswith("/uc") - - if not parsed.hostname in ("drive.google.com", "docs.google.com"): return None, is_download_link - file_id = parse_qs(parsed.query).get("id", [None])[0] - - if file_id is None: - for pattern in ( - r"^/file/d/(.*?)/(edit|view)$", - r"^/file/u/[0-9]+/d/(.*?)/(edit|view)$", - r"^/document/d/(.*?)/(edit|htmlview|view)$", - r"^/document/u/[0-9]+/d/(.*?)/(edit|htmlview|view)$", - r"^/presentation/d/(.*?)/(edit|htmlview|view)$", - r"^/presentation/u/[0-9]+/d/(.*?)/(edit|htmlview|view)$", - r"^/spreadsheets/d/(.*?)/(edit|htmlview|view)$", - r"^/spreadsheets/u/[0-9]+/d/(.*?)/(edit|htmlview|view)$" - ): - match = re.match(pattern, parsed.path) - - if match: - file_id = match.group(1) - break - - return file_id, is_download_link - -def get_url_from_gdrive_confirmation(contents): - for pattern in ( - r'href="(\/uc\?export=download[^"]+)', - r'href="/open\?id=([^"]+)"', - r'"downloadUrl":"([^"]+)' - ): - match = re.search(pattern, contents) - - if match: - url = match.group(1) - - if pattern == r'href="/open\?id=([^"]+)"': - url = ( - codecs.decode("uggcf://qevir.hfrepbagrag.tbbtyr.pbz/qbjaybnq?vq=", "rot13") + - url + - "&confirm=t&uuid=" + - re.search(r'(.*)

', contents) - if match: raise Exception(match.group(1)) - - raise Exception(translations["gdown_error"]) - -def _get_session(use_cookies, return_cookies_file=False): - sess = requests.session() - sess.headers.update({"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6)"}) - cookies_file = os.path.join(os.path.expanduser("~"), ".cache/gdown/cookies.json") - - if os.path.exists(cookies_file) and use_cookies: - with open(cookies_file) as f: - for k, v in json.load(f): - sess.cookies[k] = v - - return (sess, cookies_file) if return_cookies_file else sess - -def gdown_download(url=None, output=None): - file_id = None - if url is None: raise ValueError(translations["gdown_value_error"]) - - if "/file/d/" in url: - file_id = url.split("/d/")[1].split("/")[0] - elif "open?id=" in url: - file_id = url.split("open?id=")[1].split("/")[0] - elif "/download?id=" in url: - file_id = url.split("/download?id=")[1].split("&")[0] - - if file_id: - url = f"{codecs.decode('uggcf://qevir.tbbtyr.pbz/hp?vq=', 'rot13')}{file_id}" - url_origin = url - - sess, cookies_file = _get_session(use_cookies=True, return_cookies_file=True) - gdrive_file_id, is_gdrive_download_link = parse_url(url) - - if gdrive_file_id: - url = f"{codecs.decode('uggcf://qevir.tbbtyr.pbz/hp?vq=', 'rot13')}{gdrive_file_id}" - url_origin = url - is_gdrive_download_link = True - - while 1: - res = sess.get(url, stream=True, verify=True) - if url == url_origin and res.status_code == 500: - url = f"{codecs.decode('uggcf://qevir.tbbtyr.pbz/bcra?vq=', 'rot13')}{gdrive_file_id}" - continue - - os.makedirs(os.path.dirname(cookies_file), exist_ok=True) - with open(cookies_file, "w") as f: - json.dump( - [(k, v) for k, v in sess.cookies.items() if not k.startswith("download_warning_")], - f, - indent=2 - ) - - if ("Content-Disposition" in res.headers) or (not (gdrive_file_id and is_gdrive_download_link)): break - - try: - url = get_url_from_gdrive_confirmation(res.text) - except Exception as e: - raise Exception(e) - - if gdrive_file_id and is_gdrive_download_link: - content_disposition = unquote(res.headers["Content-Disposition"]) - - filename_from_url = ( - re.search(r"filename\*=UTF-8''(.*)", content_disposition) or re.search(r'filename=["\']?(.*?)["\']?$', content_disposition) - ).group(1).replace(os.path.sep, "_") - else: - filename_from_url = os.path.basename(url) - - output = os.path.join(output or ".", filename_from_url) - tmp_file = tempfile.mktemp(suffix=tempfile.template, prefix=os.path.basename(output), dir=os.path.dirname(output)) - f = open(tmp_file, "ab") - - if tmp_file is not None and f.tell() != 0: - res = sess.get( - url, - headers={ - "Range": f"bytes={f.tell()}-" - }, - stream=True, - verify=True - ) - - try: - with tqdm.tqdm( - desc=os.path.basename(output), - total=int(res.headers.get("Content-Length", 0)), - ncols=100, - unit="byte" - ) as pbar: - for chunk in res.iter_content(chunk_size=512 * 1024): - f.write(chunk) - pbar.update(len(chunk)) - - pbar.close() - if tmp_file: f.close() - finally: - os.rename(tmp_file, output) - sess.close() - - return output - - return None \ No newline at end of file diff --git a/main/tools/huggingface.py b/main/tools/huggingface.py deleted file mode 100644 index 19c835111f68c9bed33ae3a7b8e9fd17a7d69e2d..0000000000000000000000000000000000000000 --- a/main/tools/huggingface.py +++ /dev/null @@ -1,43 +0,0 @@ -import os -import tqdm -import requests - -try: - import wget -except: - wget = None - -def HF_download_file(url, output_path=None): - url = url.replace("/blob/", "/resolve/").replace("?download=true", "").strip() - output_path = ( - os.path.basename(url) - ) if output_path is None else ( - os.path.join(output_path, os.path.basename(url)) if os.path.isdir(output_path) else output_path - ) - - if wget != None: - wget.download( - url, - out=output_path - ) - else: - response = requests.get(url, stream=True, timeout=300) - - if response.status_code == 200: - progress_bar = tqdm.tqdm( - total=int(response.headers.get("content-length", 0)), - desc=os.path.basename(url), - ncols=100, - unit="byte", - leave=False - ) - - with open(output_path, "wb") as f: - for chunk in response.iter_content(chunk_size=10 * 1024 * 1024): - progress_bar.update(len(chunk)) - f.write(chunk) - - progress_bar.close() - else: raise ValueError(response.status_code) - - return output_path \ No newline at end of file diff --git a/main/tools/mediafire.py b/main/tools/mediafire.py deleted file mode 100644 index f339d408ea8e221ccea7446062f8fe1d1efcd59f..0000000000000000000000000000000000000000 --- a/main/tools/mediafire.py +++ /dev/null @@ -1,43 +0,0 @@ -import os -import sys -import requests - -from bs4 import BeautifulSoup - -def Mediafire_Download(url, output=None, filename=None): - if not filename: filename = url.split('/')[-2] - if not output: output = os.path.dirname(os.path.realpath(__file__)) - - output_file = os.path.join(output, filename) - sess = requests.session() - sess.headers.update({"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6)"}) - - try: - bs4 = BeautifulSoup( - sess.get(url).content, - "html.parser" - ).find(id="downloadButton").get("href") - - with requests.get( - bs4, - stream=True - ) as r: - r.raise_for_status() - - with open(output_file, "wb") as f: - total_length = int(r.headers.get('content-length')) - download_progress = 0 - - for chunk in r.iter_content(chunk_size=1024): - download_progress += len(chunk) - f.write(chunk) - - stdout = f"\r[{filename}]: {int(100 * download_progress / total_length)}% ({round(download_progress / 1024 / 1024, 2)}mb/{round(total_length / 1024 / 1024, 2)}mb)" - - sys.stdout.write(stdout) - sys.stdout.flush() - - sys.stdout.write("\n") - return output_file - except Exception as e: - raise RuntimeError(e) \ No newline at end of file diff --git a/main/tools/meganz.py b/main/tools/meganz.py deleted file mode 100644 index 6b4b830f770de91039da0fa190b26202696a761b..0000000000000000000000000000000000000000 --- a/main/tools/meganz.py +++ /dev/null @@ -1,159 +0,0 @@ -import os -import re -import sys -import json -import tqdm -import codecs -import random -import base64 -import struct -import shutil -import requests -import tempfile - -from Crypto.Cipher import AES -from Crypto.Util import Counter - -sys.path.append(os.getcwd()) - -from main.app.variables import translations - -def makebyte(x): - return codecs.latin_1_encode(x)[0] - -def a32_to_str(a): - return struct.pack('>%dI' % len(a), *a) - -def get_chunks(size): - p, s = 0, 0x20000 - - while p + s < size: - yield(p, s) - p += s - - if s < 0x100000: s += 0x20000 - - yield(p, size - p) - -def aes_cbc_decrypt(data, key): - aes_cipher = AES.new(key, AES.MODE_CBC, makebyte('\0' * 16)) - - return aes_cipher.decrypt(data) - -def decrypt_attr(attr, key): - attr = codecs.latin_1_decode(aes_cbc_decrypt(attr, a32_to_str(key)))[0].rstrip('\0') - - return json.loads(attr[4:]) if attr[:6] == 'MEGA{"' else False - -def _api_request(data): - sequence_num = random.randint(0, 0xFFFFFFFF) - params = {'id': sequence_num} - - sequence_num += 1 - if not isinstance(data, list): data = [data] - - response = requests.post( - '{0}://g.api.{1}/cs'.format('https', 'mega.co.nz'), - params=params, - data=json.dumps(data), - timeout=160 - ) - - json_resp = json.loads(response.text) - if isinstance(json_resp, int): raise Exception(json_resp) - - return json_resp[0] - -def base64_url_decode(data): - data += '=='[(2 - len(data) * 3) % 4:] - - for search, replace in (('-', '+'), ('_', '/'), (',', '')): - data = data.replace(search, replace) - - return base64.b64decode(data) - -def str_to_a32(b): - if isinstance(b, str): b = makebyte(b) - if len(b) % 4: b += b'\0' * (4 - len(b) % 4) - - return struct.unpack('>%dI' % (len(b) / 4), b) - -def base64_to_a32(s): - return str_to_a32( - base64_url_decode(s) - ) - -def mega_download_file(file_handle, file_key, dest_path=None): - file_key = base64_to_a32(file_key) - file_data = _api_request({'a': 'g', 'g': 1, 'p': file_handle}) - - k = ( - file_key[0] ^ file_key[4], - file_key[1] ^ file_key[5], - file_key[2] ^ file_key[6], - file_key[3] ^ file_key[7] - ) - - iv = file_key[4:6] + (0, 0) - if 'g' not in file_data: raise Exception(translations["file_not_access"]) - - file_size = file_data['s'] - attribs = decrypt_attr(base64_url_decode(file_data['at']), k) - - input_file = requests.get(file_data['g'], stream=True).raw - temp_output_file = tempfile.NamedTemporaryFile(mode='w+b', prefix='megapy_', delete=False) - - k_str = a32_to_str(k) - aes = AES.new( - k_str, - AES.MODE_CTR, - counter=Counter.new( - 128, - initial_value=( - (iv[0] << 32) + iv[1] - ) << 64 - ) - ) - - mac_str = b'\0' * 16 - mac_encryptor = AES.new(k_str, AES.MODE_CBC, mac_str) - iv_str = a32_to_str([iv[0], iv[1], iv[0], iv[1]]) - - with tqdm.tqdm(total=file_size, ncols=100, unit="byte") as pbar: - for _, chunk_size in get_chunks(file_size): - chunk = aes.decrypt(input_file.read(chunk_size)) - temp_output_file.write(chunk) - - pbar.update(len(chunk)) - encryptor = AES.new(k_str, AES.MODE_CBC, iv_str) - - for i in range(0, len(chunk) - 16, 16): - block = chunk[i:i + 16] - encryptor.encrypt(block) - - i = (i + 16) if file_size > 16 else 0 - block = chunk[i:i + 16] - - if len(block) % 16: block += b'\0' * (16 - (len(block) % 16)) - mac_str = mac_encryptor.encrypt(encryptor.encrypt(block)) - - file_mac = str_to_a32(mac_str) - temp_output_file.close() - - if (file_mac[0] ^ file_mac[1], file_mac[2] ^ file_mac[3]) != file_key[6:8]: raise ValueError(translations["mac_not_match"]) - - file_path = os.path.join(dest_path, attribs['n']) - if os.path.exists(file_path): os.remove(file_path) - - shutil.move(temp_output_file.name, file_path) - return file_path - -def mega_download_url(url, dest_path=None): - if '/file/' in url: - url = url.replace(' ', '') - file_id = re.findall(r'\W\w\w\w\w\w\w\w\w\W', url)[0][1:-1] - path = f'{file_id}!{url[re.search(file_id, url).end() + 1:]}'.split('!') - elif '!' in url: path = re.findall(r'/#!(.*)', url)[0].split('!') - else: raise Exception(translations["missing_url"]) - - return mega_download_file(path[0], path[1], dest_path) \ No newline at end of file diff --git a/main/tools/pixeldrain.py b/main/tools/pixeldrain.py deleted file mode 100644 index c20798850bad46ebbc9e241b82fe86a27e48a1a1..0000000000000000000000000000000000000000 --- a/main/tools/pixeldrain.py +++ /dev/null @@ -1,21 +0,0 @@ -import os -import requests - -def pixeldrain(url, output_dir): - try: - response = requests.get(f"https://pixeldrain.com/api/file/{url.split('pixeldrain.com/u/')[1]}") - - if response.status_code == 200: - file_path = os.path.join( - output_dir, - response.headers.get("Content-Disposition").split("filename=")[-1].strip('";') - ) - - with open(file_path, "wb") as newfile: - newfile.write(response.content) - - return file_path - - return None - except Exception as e: - raise RuntimeError(e) \ No newline at end of file diff --git a/main/tools/sortedcontainers.py b/main/tools/sortedcontainers.py deleted file mode 100644 index b60fbdc6c64c2542aa92c730a162ecd86431fd92..0000000000000000000000000000000000000000 --- a/main/tools/sortedcontainers.py +++ /dev/null @@ -1,752 +0,0 @@ -import os -import sys -import math -import bisect -import operator -import collections.abc - -from _thread import get_ident -from functools import reduce, wraps -from itertools import chain, repeat, starmap - -sys.path.append(os.getcwd()) - -from main.app.variables import logger - -def recursive_repr(func): - repr_running = set() - - @wraps(func) - def wrapper(self): - key = id(self), get_ident() - if key in repr_running: return '...' - - repr_running.add(key) - - try: - return func(self) - finally: - repr_running.discard(key) - - return wrapper - -class SortedList(collections.abc.MutableSequence): - def __init__(self, iterable=None, load=1000): - self._len, self._maxes, self._lists, self._index = 0, [], [], [] - self._load, self._twice, self._half = load, load * 2, load >> 1 - if iterable is not None: self.update(iterable) - - def clear(self): - self._len = 0 - - del self._maxes[:] - del self._lists[:] - del self._index[:] - - def add(self, val): - _maxes, _lists = self._maxes, self._lists - - if _maxes: - pos = bisect.bisect_right(_maxes, val) - - if pos == len(_maxes): - pos -= 1 - _maxes[pos] = val - _lists[pos].append(val) - else: - bisect.insort(_lists[pos], val) - - self._expand(pos) - else: - _maxes.append(val) - _lists.append([val]) - - self._len += 1 - - def _expand(self, pos): - _lists, _index = self._lists, self._index - - if len(_lists[pos]) > self._twice: - _maxes, _load = self._maxes, self._load - half = _lists[pos][_load:] - - _lists[pos] = _lists[pos][:_load] - _maxes[pos] = _lists[pos][-1] - - _maxes.insert(pos + 1, half[-1]) - _lists.insert(pos + 1, half) - - del _index[:] - else: - if len(_index) > 0: - child = self._offset + pos - - while child > 0: - _index[child] += 1 - child = (child - 1) >> 1 - - _index[0] += 1 - - def update(self, iterable): - _maxes, _lists = self._maxes, self._lists - values = sorted(iterable) - - if _maxes: - if len(values) * 4 >= self._len: - values.extend(chain.from_iterable(_lists)) - values.sort() - self.clear() - else: - _add = self.add - - for val in values: - _add(val) - - return - - _load, _index = self._load, self._index - _lists.extend(values[pos:(pos + _load)] for pos in range(0, len(values), _load)) - _maxes.extend(sublist[-1] for sublist in _lists) - - self._len = len(values) - del _index[:] - - def __contains__(self, val): - _maxes = self._maxes - if not _maxes: return False - - pos = bisect.bisect_left(_maxes, val) - if pos == len(_maxes): return False - - _lists = self._lists - idx = bisect.bisect_left(_lists[pos], val) - return _lists[pos][idx] == val - - def discard(self, val): - _maxes = self._maxes - - if not _maxes: return - pos = bisect.bisect_left(_maxes, val) - - if pos == len(_maxes): return - _lists = self._lists - - idx = bisect.bisect_left(_lists[pos], val) - if _lists[pos][idx] == val: self._delete(pos, idx) - - def remove(self, val): - _maxes = self._maxes - if not _maxes: raise ValueError - - pos = bisect.bisect_left(_maxes, val) - if pos == len(_maxes): raise ValueError - - _lists = self._lists - idx = bisect.bisect_left(_lists[pos], val) - - if _lists[pos][idx] == val: self._delete(pos, idx) - else: raise ValueError - - def _delete(self, pos, idx): - _maxes, _lists, _index = self._maxes, self._lists, self._index - - lists_pos = _lists[pos] - del lists_pos[idx] - - self._len -= 1 - len_lists_pos = len(lists_pos) - - if len_lists_pos > self._half: - _maxes[pos] = lists_pos[-1] - - if len(_index) > 0: - child = self._offset + pos - - while child > 0: - _index[child] -= 1 - child = (child - 1) >> 1 - - _index[0] -= 1 - elif len(_lists) > 1: - if pos == 0: pos += 1 - - prev = pos - 1 - _lists[prev].extend(_lists[pos]) - _maxes[prev] = _lists[prev][-1] - - del _maxes[pos] - del _lists[pos] - del _index[:] - - self._expand(prev) - elif len_lists_pos: _maxes[pos] = lists_pos[-1] - else: - del _maxes[pos] - del _lists[pos] - del _index[:] - - def _loc(self, pos, idx): - if pos == 0: return idx - - _index = self._index - if len(_index) == 0: self._build_index() - - total = 0 - pos += self._offset - - while pos: - if not (pos & 1): total += _index[pos - 1] - pos = (pos - 1) >> 1 - - return total + idx - - def _pos(self, idx): - _len, _lists = self._len, self._lists - - if idx < 0: - last_len = len(_lists[-1]) - if (-idx) <= last_len: return len(_lists) - 1, last_len + idx - - idx += _len - if idx < 0: raise IndexError - elif idx >= _len: raise IndexError - - if idx < len(_lists[0]): return 0, idx - - _index = self._index - if len(_index) == 0: self._build_index() - - pos = 0 - len_index = len(_index) - child = (pos << 1) + 1 - - while child < len_index: - index_child = _index[child] - - if idx < index_child: pos = child - else: - idx -= index_child - pos = child + 1 - - child = (pos << 1) + 1 - - return (pos - self._offset, idx) - - def _build_index(self): - row0 = list(map(len, self._lists)) - - if len(row0) == 1: - self._index[:] = row0 - self._offset = 0 - return - - head = iter(row0) - tail = iter(head) - row1 = list(starmap(operator.add, zip(head, tail))) - - if len(row0) & 1: row1.append(row0[-1]) - - if len(row1) == 1: - self._index[:] = row1 + row0 - self._offset = 1 - return - - size = 2 ** (int(math.log(len(row1) - 1, 2)) + 1) - row1.extend(repeat(0, size - len(row1))) - tree = [row0, row1] - - while len(tree[-1]) > 1: - head = iter(tree[-1]) - tail = iter(head) - - row = list(starmap(operator.add, zip(head, tail))) - tree.append(row) - - reduce(operator.iadd, reversed(tree), self._index) - self._offset = size * 2 - 1 - - def _slice(self, slc): - start, stop, step = slc.start, slc.stop, slc.step - if step == 0: raise ValueError - - if step is None: step = 1 - - if step > 0: - if start is None: start = 0 - if stop is None: stop = len(self) - elif stop < 0: stop += len(self) - else: - if start is None: start = len(self) - 1 - if stop is None: stop = -1 - elif stop < 0: stop += len(self) - - if start < 0: start += len(self) - - if step > 0: - if start < 0: start = 0 - elif start > len(self): start = len(self) - if stop < 0: stop = 0 - elif stop > len(self): stop = len(self) - else: - if start < 0: start = -1 - elif start >= len(self): start = len(self) - 1 - if stop < 0: stop = -1 - elif stop > len(self): stop = len(self) - - return start, stop, step - - def __delitem__(self, idx): - if isinstance(idx, slice): - start, stop, step = self._slice(idx) - - if ((step == 1) and (start < stop) and ((stop - start) * 8 >= self._len)): - values = self[:start] - if stop < self._len: values += self[stop:] - - self.clear() - self.update(values) - - return - - indices = range(start, stop, step) - if step > 0: indices = reversed(indices) - _pos, _delete = self._pos, self._delete - - for index in indices: - pos, idx = _pos(index) - _delete(pos, idx) - else: - pos, idx = self._pos(idx) - self._delete(pos, idx) - - def __getitem__(self, idx): - _lists = self._lists - - if isinstance(idx, slice): - start, stop, step = self._slice(idx) - if step == 1 and start < stop: - if start == 0 and stop == self._len: return self.as_list() - start_pos, start_idx = self._pos(start) - - if stop == self._len: - stop_pos = len(_lists) - 1 - stop_idx = len(_lists[stop_pos]) - else: stop_pos, stop_idx = self._pos(stop) - - if start_pos == stop_pos: return _lists[start_pos][start_idx:stop_idx] - - prefix = _lists[start_pos][start_idx:] - middle = _lists[(start_pos + 1):stop_pos] - - result = reduce(operator.iadd, middle, prefix) - result += _lists[stop_pos][:stop_idx] - - return result - - if step == -1 and start > stop: - result = self[(stop + 1):(start + 1)] - result.reverse() - return result - - indices = range(start, stop, step) - return list(self[index] for index in indices) - else: - pos, idx = self._pos(idx) - return _lists[pos][idx] - - def _check_order(self, idx, val): - _lists, _len = self._lists, self._len - pos, loc = self._pos(idx) - if idx < 0: idx += _len - - if idx > 0: - idx_prev = loc - 1 - pos_prev = pos - - if idx_prev < 0: - pos_prev -= 1 - idx_prev = len(_lists[pos_prev]) - 1 - - if _lists[pos_prev][idx_prev] > val: raise ValueError - - if idx < (_len - 1): - idx_next = loc + 1 - pos_next = pos - - if idx_next == len(_lists[pos_next]): - pos_next += 1 - idx_next = 0 - - if _lists[pos_next][idx_next] < val: raise ValueError - - def __setitem__(self, index, value): - _maxes, _lists, _pos = self._maxes, self._lists, self._pos - _check_order = self._check_order - - if isinstance(index, slice): - start, stop, step = self._slice(index) - indices = range(start, stop, step) - - if step != 1: - if not hasattr(value, '__len__'): value = list(value) - - indices = list(indices) - if len(value) != len(indices): raise ValueError - - log = [] - _append = log.append - - for idx, val in zip(indices, value): - pos, loc = _pos(idx) - _append((idx, _lists[pos][loc], val)) - - _lists[pos][loc] = val - if len(_lists[pos]) == (loc + 1): _maxes[pos] = val - - try: - for idx, oldval, newval in log: - _check_order(idx, newval) - except ValueError: - for idx, oldval, newval in log: - pos, loc = _pos(idx) - _lists[pos][loc] = oldval - - if len(_lists[pos]) == (loc + 1): _maxes[pos] = oldval - - raise - else: - if not hasattr(value, '__getitem__'): value = list(value) - ordered = all(value[pos - 1] <= value[pos] for pos in range(1, len(value))) - if not ordered: raise ValueError - - if start == 0 or len(value) == 0: pass - else: - if self[start - 1] > value[0]: raise ValueError - - if stop == len(self) or len(value) == 0: pass - else: - if self[stop] < value[-1]: raise ValueError - - del self[index] - _insert = self.insert - - for idx, val in enumerate(value): - _insert(start + idx, val) - else: - pos, loc = _pos(index) - _check_order(index, value) - - _lists[pos][loc] = value - if len(_lists[pos]) == (loc + 1): _maxes[pos] = value - - def __iter__(self): - return chain.from_iterable(self._lists) - - def __reversed__(self): - _lists = self._lists - start = len(_lists) - 1 - - iterable = (reversed(_lists[pos]) for pos in range(start, -1, -1)) - return chain.from_iterable(iterable) - - def __len__(self): - return self._len - - def bisect_left(self, val): - _maxes = self._maxes - if not _maxes: return 0 - - pos = bisect.bisect_left(_maxes, val) - if pos == len(_maxes): return self._len - - idx = bisect.bisect_left(self._lists[pos], val) - return self._loc(pos, idx) - - def bisect(self, val): - return self.bisect_right(val) - - def bisect_right(self, val): - _maxes = self._maxes - if not _maxes: return 0 - - pos = bisect.bisect_right(_maxes, val) - if pos == len(_maxes): return self._len - idx = bisect.bisect_right(self._lists[pos], val) - - return self._loc(pos, idx) - - def count(self, val): - _maxes = self._maxes - if not _maxes: return 0 - - pos_left = bisect.bisect_left(_maxes, val) - if pos_left == len(_maxes): return 0 - - _lists = self._lists - idx_left = bisect.bisect_left(_lists[pos_left], val) - pos_right = bisect.bisect_right(_maxes, val) - - if pos_right == len(_maxes): return self._len - self._loc(pos_left, idx_left) - idx_right = bisect.bisect_right(_lists[pos_right], val) - - if pos_left == pos_right: - return idx_right - idx_left - - right = self._loc(pos_right, idx_right) - left = self._loc(pos_left, idx_left) - - return right - left - - def copy(self): - return SortedList(self, load=self._load) - - def __copy__(self): - return self.copy() - - def append(self, val): - _maxes, _lists = self._maxes, self._lists - - if not _maxes: - _maxes.append(val) - _lists.append([val]) - - self._len = 1 - return - - pos = len(_lists) - 1 - if val < _lists[pos][-1]: raise ValueError - - _maxes[pos] = val - _lists[pos].append(val) - - self._len += 1 - self._expand(pos) - - def extend(self, values): - _maxes, _lists, _load = self._maxes, self._lists, self._load - - if not isinstance(values, list): - values = list(values) - - if any(values[pos - 1] > values[pos] - for pos in range(1, len(values))): - raise ValueError - - offset = 0 - - if _maxes: - if values[0] < _lists[-1][-1]: raise ValueError - - if len(_lists[-1]) < self._half: - _lists[-1].extend(values[:_load]) - _maxes[-1] = _lists[-1][-1] - offset = _load - - len_lists = len(_lists) - - for idx in range(offset, len(values), _load): - _lists.append(values[idx:(idx + _load)]) - _maxes.append(_lists[-1][-1]) - - _index = self._index - - if len_lists == len(_lists): - len_index = len(_index) - - if len_index > 0: - len_values = len(values) - child = len_index - 1 - - while child: - _index[child] += len_values - child = (child - 1) >> 1 - - _index[0] += len_values - else: del self._index[:] - - self._len += len(values) - - def insert(self, idx, val): - _maxes, _lists, _len = self._maxes, self._lists, self._len - - if idx < 0: idx += _len - if idx < 0: idx = 0 - if idx > _len: idx = _len - - if not _maxes: - _maxes.append(val) - _lists.append([val]) - - self._len = 1 - return - - if idx == 0: - if val > _lists[0][0]: raise ValueError - else: - _lists[0].insert(0, val) - self._expand(0) - - self._len += 1 - return - - if idx == _len: - pos = len(_lists) - 1 - - if _lists[pos][-1] > val: raise ValueError - else: - _lists[pos].append(val) - _maxes[pos] = _lists[pos][-1] - - self._expand(pos) - self._len += 1 - return - - pos, idx = self._pos(idx) - idx_before = idx - 1 - - if idx_before < 0: - pos_before = pos - 1 - idx_before = len(_lists[pos_before]) - 1 - else: pos_before = pos - - before = _lists[pos_before][idx_before] - - if before <= val <= _lists[pos][idx]: - _lists[pos].insert(idx, val) - self._expand(pos) - self._len += 1 - else: raise ValueError - - def pop(self, idx=-1): - if (idx < 0 and -idx > self._len) or (idx >= self._len): raise IndexError - - pos, idx = self._pos(idx) - val = self._lists[pos][idx] - self._delete(pos, idx) - - return val - - def index(self, val, start=None, stop=None): - _len, _maxes = self._len, self._maxes - if not _maxes: raise ValueError - - if start is None: start = 0 - if start < 0: start += _len - if start < 0: start = 0 - if stop is None: stop = _len - if stop < 0: stop += _len - if stop > _len: stop = _len - - if stop <= start: raise ValueError - - stop -= 1 - pos_left = bisect.bisect_left(_maxes, val) - if pos_left == len(_maxes): raise ValueError - - _lists = self._lists - idx_left = bisect.bisect_left(_lists[pos_left], val) - - if _lists[pos_left][idx_left] != val: raise ValueError - left = self._loc(pos_left, idx_left) - - if start <= left: - if left <= stop: return left - else: - right = self.bisect_right(val) - 1 - if start <= right: return start - - raise ValueError - - def as_list(self): - return reduce(operator.iadd, self._lists, []) - - def __add__(self, that): - values = self.as_list() - values.extend(that) - - return SortedList(values) - - def __iadd__(self, that): - self.update(that) - return self - - def __mul__(self, that): - values = self.as_list() * that - return SortedList(values) - - def __imul__(self, that): - values = self.as_list() * that - self.clear() - self.update(values) - return self - - def __eq__(self, that): - return ((self._len == len(that)) and all(lhs == rhs for lhs, rhs in zip(self, that))) - - def __ne__(self, that): - return ((self._len != len(that)) or any(lhs != rhs for lhs, rhs in zip(self, that))) - - def __lt__(self, that): - return ((self._len <= len(that)) and all(lhs < rhs for lhs, rhs in zip(self, that))) - - def __le__(self, that): - return ((self._len <= len(that)) and all(lhs <= rhs for lhs, rhs in zip(self, that))) - - def __gt__(self, that): - return ((self._len >= len(that)) and all(lhs > rhs for lhs, rhs in zip(self, that))) - - def __ge__(self, that): - return ((self._len >= len(that)) and all(lhs >= rhs for lhs, rhs in zip(self, that))) - - @recursive_repr - def __repr__(self): - return '{0}({1})'.format(self.__class__.__name__, repr(self.as_list())) - - def _check(self): - try: - assert self._load >= 4 - assert self._half == (self._load >> 1) - assert self._twice == (self._load * 2) - - if self._maxes == []: - assert self._lists == [] - return - - assert len(self._maxes) > 0 and len(self._lists) > 0 - assert all(sublist[pos - 1] <= sublist[pos] for sublist in self._lists for pos in range(1, len(sublist))) - - for pos in range(1, len(self._lists)): - assert self._lists[pos - 1][-1] <= self._lists[pos][0] - - assert len(self._maxes) == len(self._lists) - assert all(self._maxes[pos] == self._lists[pos][-1] for pos in range(len(self._maxes))) - assert all(len(sublist) <= self._twice for sublist in self._lists) - assert all(len(self._lists[pos]) >= self._half for pos in range(0, len(self._lists) - 1)) - assert self._len == sum(len(sublist) for sublist in self._lists) - - if len(self._index): - assert len(self._index) == self._offset + len(self._lists) - assert self._len == self._index[0] - - def test_offset_pos(pos): - from_index = self._index[self._offset + pos] - return from_index == len(self._lists[pos]) - - assert all(test_offset_pos(pos) for pos in range(len(self._lists))) - - for pos in range(self._offset): - child = (pos << 1) + 1 - - if self._index[pos] == 0: assert child >= len(self._index) - elif child + 1 == len(self._index): assert self._index[pos] == self._index[child] - else: assert self._index[pos] == (self._index[child] + self._index[child + 1]) - except: - import traceback - - logger.error(traceback.format_exc()) - logger.debug(self._len, self._load, self._half, self._twice) - logger.debug(self._index) - logger.debug(self._maxes) - logger.debug(self._lists) - - raise \ No newline at end of file