Spaces:
Running
Running
| import copy, json, threading | |
| import subprocess | |
| import platform, glob, sys | |
| import math | |
| import os | |
| import re | |
| import shutil | |
| import time | |
| from concurrent.futures import ThreadPoolExecutor | |
| from dataclasses import dataclass, field | |
| from pathlib import Path | |
| from typing import List, Dict, Union | |
| from videotrans import translator | |
| from videotrans.configure.config import ROOT_DIR, tr, app_cfg, settings, logger | |
| from videotrans.configure import config | |
| from videotrans.recognition import run as run_recogn, is_allow_lang as recogn_allow_lang, FASTER_WHISPER | |
| from videotrans.translator import run as run_trans, get_audio_code | |
| from videotrans.tts import run as run_tts, EDGE_TTS, AZURE_TTS, SUPPORT_CLONE | |
| from videotrans.task.simple_runnable_qt import run_in_threadpool | |
| from ..configure import contants | |
| from ._base import BaseTask | |
| from videotrans.util.help_ffmpeg import get_video_codec, get_audio_time, runffmpeg, get_video_info, conver_to_16k, \ | |
| get_video_duration, cut_from_audio, create_concat_txt, concat_multi_audio, change_speed_rubberband | |
| from videotrans.task.taskcfg import TaskCfgVTT | |
| from videotrans.configure.excepts import VideoTransError, FFmpegError, SpeechToTextError, DubbingSrtError | |
| from videotrans.util.help_misc import vail_file, read_last_n_lines, is_novoice_mp4, get_md5 | |
| from videotrans.util.help_srt import get_subtitle_from_srt, delete_punc, ms_to_time_string, simple_wrap, set_ass_font | |
| class TransCreate(BaseTask): | |
| cfg: TaskCfgVTT = field(default_factory=TaskCfgVTT, repr=False) | |
| # 存放原始语言字幕 | |
| source_srt_list: List = field(default_factory=list) | |
| # 存放目标语言字幕 | |
| target_srt_list: List = field(default_factory=list) | |
| # 原始视频时长 在慢速处理合并后,时长更新至此 | |
| video_time: float = 0.0 | |
| # 视频信息 | |
| """ | |
| { | |
| "video_fps":0, | |
| "video_codec_name":"h264", | |
| "audio_codec_name":"aac", | |
| "width":0, | |
| "height":0, | |
| "time":0 | |
| } | |
| """ | |
| video_info: Dict = field(default_factory=dict, repr=False) | |
| # 对视频是否执行 c:v copy 操作 | |
| is_copy_video: bool = False | |
| # 需要输出的mp4编码类型 264 265 | |
| video_codec_num: int = 264 | |
| # 是否忽略音频和视频对齐 | |
| ignore_align: bool = False | |
| # 是否是音频翻译任务,如果是,则到配音完毕即结束,无需合并 | |
| is_audio_trans: bool = False | |
| queue_tts: List = field(default_factory=list, repr=False) | |
| clone_ref: str = "" | |
| cost_duration:float=0.0 | |
| should_recogn2:bool=False | |
| def __post_init__(self): | |
| super().__post_init__() | |
| self.cost_duration=time.time() | |
| if not self.cfg.cache_folder: | |
| self.cfg.cache_folder = f"{config.TEMP_DIR}/{self.uuid}" | |
| # 清理缓存 | |
| if self.cfg.clear_cache: | |
| if self.cfg.target_dir and Path(self.cfg.target_dir).is_dir(): | |
| shutil.rmtree(self.cfg.target_dir, ignore_errors=True) | |
| if self.cfg.cache_folder and Path(self.cfg.cache_folder).is_dir(): | |
| shutil.rmtree(self.cfg.cache_folder, ignore_errors=True) | |
| self.signal(text=tr('kaishichuli')) | |
| # -1=不启用说话人,0=启用并且不限制说话人数量,>0+1 为最大说话人数量 | |
| self.max_speakers = self.cfg.nums_diariz if self.cfg.enable_diariz else -1 | |
| if self.max_speakers > 0: | |
| self.max_speakers += 1 | |
| self.should_recogn = True | |
| # 输出编码, 264 或 265 | |
| self.video_codec_num = int(settings.get('video_codec', 264)) | |
| # 输出文件夹,去掉可能存在的双斜线 | |
| #self.cfg.target_dir = re.sub(r'/{2,}', '/', self.cfg.target_dir, flags=re.I | re.S) | |
| # 检测字幕原始语言 | |
| self.cfg.detect_language = get_audio_code(show_source=self.cfg.source_language_code) | |
| # 存放分离后的无声mp4到临时文件夹 | |
| self.cfg.novoice_mp4 = f"{self.cfg.cache_folder}/novoice.mp4" | |
| # 原始语言字幕文件:输出文件夹 | |
| self.cfg.source_sub = f"{self.cfg.target_dir}/{self.cfg.source_language_code}.srt" | |
| # 原始语言音频文件:输出文件夹 | |
| self.cfg.source_wav_output = f"{self.cfg.target_dir}/{self.cfg.source_language_code}.m4a" | |
| # 原始语言音频文件:临时文件夹 | |
| self.cfg.source_wav = f"{self.cfg.cache_folder}/{self.cfg.source_language_code}.wav" | |
| # 目标语言字幕:输出文件夹 | |
| self.cfg.target_sub = f"{self.cfg.target_dir}/{self.cfg.target_language_code}.srt" | |
| # 配音后的目标音频文件:输出文件夹 | |
| self.cfg.target_wav_output = f"{self.cfg.target_dir}/{self.cfg.target_language_code}.m4a" | |
| # 配音后的目标音频文件:临时文件夹 | |
| self.cfg.target_wav = f"{self.cfg.cache_folder}/target.wav" | |
| # 最终需要输出的mp4视频 | |
| self.cfg.targetdir_mp4 = f"{self.cfg.target_dir}/{self.cfg.noextname}.mp4" | |
| # 如果配音角色不是No 则需要配音 | |
| if self.cfg.voice_role and self.cfg.voice_role != 'No' and self.cfg.target_language_code: | |
| self.should_dubbing = True | |
| # 如果不是 tiqu,则均需要合并视频音频字幕 | |
| if self.cfg.app_mode != 'tiqu' and (self.should_dubbing or self.cfg.subtitle_type > 0): | |
| self.should_hebing = True | |
| # 是否需要翻译:存在目标语言代码并且不等于原始语言,则需要翻译 | |
| if self.cfg.target_language_code and self.cfg.target_language_code != self.cfg.source_language_code: | |
| self.should_trans = True | |
| # 如果原语言和目标语言相等,并且存在配音角色,则替换配音 | |
| if self.cfg.voice_role and self.cfg.voice_role != 'No' and self.cfg.source_language_code == self.cfg.target_language_code: | |
| self.cfg.target_wav_output = f"{self.cfg.target_dir}/{self.cfg.target_language_code}-dubbing.m4a" | |
| self.cfg.target_wav = f"{self.cfg.cache_folder}/target-dubbing.wav" | |
| self.should_dubbing = True | |
| # 判断如果是音频,则到生成音频结束,无需合并,并且无需分离视频、无需背景音处理 | |
| if self.cfg.ext in contants.AUDIO_EXITS: | |
| self.is_audio_trans = True | |
| # self.cfg.is_separate = False | |
| self.should_hebing = False | |
| # 没有设置目标语言,不配音不翻译 | |
| if not self.cfg.target_language_code: | |
| self.should_dubbing = False | |
| self.should_trans = False | |
| if self.cfg.voice_role == 'No': | |
| self.should_dubbing = False | |
| if self.cfg.app_mode == 'tiqu': | |
| self.cfg.enable_diariz = False | |
| self.should_dubbing = False | |
| self.should_separate = self.cfg.is_separate | |
| # 是否需要二次识别 | |
| # 选中二次识别 and 有配音 and 非嵌入双字幕 and 有翻译即原始和目标语言非同一个 | |
| self.should_recogn2 = self.cfg.recogn2pass and self.should_dubbing and self.cfg.subtitle_type<3 and (self.cfg.source_language_code != self.cfg.target_language_code) | |
| self.cfg.vocal = f"{self.cfg.cache_folder}/vocal.wav" | |
| self.cfg.instrument = f"{self.cfg.cache_folder}/instrument.wav" | |
| # 记录最终使用的配置信息 | |
| logger.debug(f"[TransCreate]最终配置信息:{self=}\n{self.cfg=}") | |
| # 禁止修改字幕 | |
| self.signal(text="forbid", type="disabled_edit") | |
| # 开启一个线程计时,避免看起来卡死 | |
| def runing(): | |
| t = time.time() | |
| while not self.hasend: | |
| if self._exit(): return | |
| time.sleep(1) | |
| self.signal(text=f"{int(time.time() - t)}???{self.precent}", type="set_precent") | |
| if app_cfg.exec_mode != 'cli': | |
| threading.Thread(target=runing, daemon=True).start() | |
| # 1. 预处理,分离音视频、分离人声等 | |
| def prepare(self) -> None: | |
| _st=time.time() | |
| if self._exit(): return | |
| self.signal(text=tr("Hold on a monment...")) | |
| Path(self.cfg.cache_folder).mkdir(parents=True, exist_ok=True) | |
| Path(self.cfg.target_dir).mkdir(parents=True, exist_ok=True) | |
| # 删掉可能存在的无效文件 | |
| self._unlink_size0([self.cfg.source_sub, self.cfg.target_sub, self.cfg.targetdir_mp4]) | |
| # 获取视频信息 | |
| self.video_info = get_video_info(self.cfg.name) | |
| # 视频时长,毫秒 | |
| self.video_time = self.video_info['time'] | |
| # 音频时长,毫秒 | |
| audio_stream_len = self.video_info.get('streams_audio', 0) | |
| # 无视频流,不是音频,并且不是提取,报错 | |
| if self.video_info.get('video_streams', 0) < 1 and not self.is_audio_trans and self.cfg.app_mode != 'tiqu': | |
| raise VideoTransError( | |
| tr('The video file {} does not contain valid video data and cannot be processed.', self.cfg.name)) | |
| # 无音频流,不存在原语言字幕,报错。存在则是无声视频流 | |
| if audio_stream_len < 1 and not vail_file(self.cfg.source_sub): | |
| raise VideoTransError( | |
| tr('There is no valid audio in the file {} and it cannot be processed. Please play it manually to confirm that there is sound.', | |
| self.cfg.name)) | |
| # 如果获得原始视频编码格式是 h264,并且色素 yuv420p, 则直接复制视频流 is_copy_video=True | |
| if self.video_info['video_codec_name'] == 'h264' and self.video_info['color'] == 'yuv420p': | |
| self.is_copy_video = True | |
| # 如果存在字幕文本,则视为原始语言字幕,不再识别 | |
| if self.cfg.subtitles.strip(): | |
| with open(self.cfg.source_sub, 'w', encoding="utf-8", errors="ignore") as f: | |
| txt = re.sub(r':\d+\.\d+', lambda m: m.group().replace('.', ','), | |
| self.cfg.subtitles.strip(), flags=re.I | re.S) | |
| f.write(txt) | |
| self.should_recogn = False | |
| # 判断是否已存在人声文件,只要存在, 即使用此文件作为语音识别原料 | |
| raw_vocal = f"{self.cfg.target_dir}/vocal.wav" | |
| if vail_file(raw_vocal): | |
| shutil.copy2(raw_vocal, self.cfg.vocal) | |
| raw_instrument = f"{self.cfg.target_dir}/instrument.wav" | |
| if vail_file(raw_instrument): | |
| shutil.copy2(raw_instrument, self.cfg.instrument) | |
| # 将原始视频分离为无声视频 | |
| if not self.is_audio_trans and self.cfg.app_mode != 'tiqu': | |
| app_cfg.queue_novice[self.uuid] = 'ing' | |
| if not self.is_copy_video: | |
| self.signal(text=tr("Video needs transcoded and take a long time..")) | |
| run_in_threadpool(self._split_novoice_byraw) | |
| else: | |
| app_cfg.queue_novice[self.uuid] = 'end' | |
| # 需要人声背景声分离,并且不存在已分离好的文件 | |
| if audio_stream_len > 0 and self.cfg.is_separate and ( | |
| not vail_file(self.cfg.vocal) or not vail_file(self.cfg.instrument)): | |
| self.signal(text=tr('Separating background music')) | |
| try: | |
| self._split_audio_byraw(True) | |
| except Exception as e: | |
| logger.exception(f'分离人声背景声失败,跳过 {e}', exc_info=True) | |
| finally: | |
| if not vail_file(self.cfg.vocal) or not vail_file(self.cfg.instrument): | |
| # 分离失败 | |
| self.cfg.is_separate = self.should_separate = False | |
| if audio_stream_len > 0 and not vail_file(self.cfg.source_wav) and vail_file(self.cfg.vocal): | |
| # 如果存在人声文件(可能仅仅分离成功人声,或者单独将其他工具分离出的人声放入目标文件夹),则使用该文件作为语音识别文件 | |
| cmd = [ | |
| "-y", | |
| "-i", | |
| self.cfg.vocal, | |
| "-ac", | |
| "1", | |
| "-ar", | |
| "16000", | |
| "-c:a", | |
| "pcm_s16le", | |
| self.cfg.source_wav | |
| ] | |
| try: | |
| logger.debug(f'存在单独的人声文件 vocal.wav, 使用此作为语音识别原始音频') | |
| runffmpeg(cmd) | |
| except Exception as e: | |
| logger.exception(f'将 人声文件 转为 16000 source_wav 时失败 {e}', exc_info=True) | |
| # 如果还不存在原音频 self.cfg.source_wav,说明失败,强制从原视频中提取 | |
| if audio_stream_len > 0 and not vail_file(self.cfg.source_wav): | |
| self._split_audio_byraw() | |
| # 将分离后人声设为语音克隆参考音频 | |
| if self.cfg.vocal and Path(self.cfg.vocal).exists(): | |
| self.clone_ref = self.cfg.vocal | |
| self.signal(text=tr('endfenliyinpin')) | |
| logger.debug(f'[预处理阶段结束耗时]:{time.time()-_st}s') | |
| # 开始识别 | |
| def recogn(self) -> None: | |
| _st=time.time() | |
| if self._exit(): return | |
| if not self.should_recogn: return | |
| self.precent += 3 | |
| self.signal(text=tr("kaishishibie")) | |
| if vail_file(self.cfg.source_sub): | |
| self.source_srt_list = get_subtitle_from_srt(self.cfg.source_sub, is_file=True) | |
| if Path(self.cfg.target_dir + "/speaker.json").exists(): | |
| shutil.copy2(self.cfg.target_dir + "/speaker.json", self.cfg.cache_folder + "/speaker.json") | |
| self._recogn_succeed() | |
| return | |
| if not vail_file(self.cfg.source_wav): | |
| raise SpeechToTextError(tr("Failed to separate audio, please check the log or retry")) | |
| from videotrans.util.help_down import down_file_from_ms | |
| # 进行降噪,结果为 16k采样 | |
| if self.cfg.remove_noise: | |
| _remove_noise_wav = f"{self.cfg.cache_folder}/remove_noise.wav" | |
| if vail_file(_remove_noise_wav): | |
| self.cfg.source_wav = _remove_noise_wav | |
| self.clone_ref = _remove_noise_wav | |
| logger.debug(f'复用已存在的降噪缓存文件') | |
| else: | |
| title = tr("Starting to process speech noise reduction, which may take a long time, please be patient") | |
| down_file_from_ms(f'{ROOT_DIR}/models/onnx', urls=[ | |
| 'https://modelscope.cn/models/himyworld/videotrans/resolve/master/onnx/dpdfnet4.onnx'], | |
| callback=self._process_callback) | |
| from videotrans.process.prepare_audio import remove_noise | |
| kw = { | |
| "input_file": self.cfg.source_wav if not self.cfg.vocal or not Path(self.cfg.vocal).exists() else self.cfg.vocal, | |
| "output_file": _remove_noise_wav, | |
| "is_cuda": self.cfg.is_cuda | |
| } | |
| try: | |
| _rs = self._new_process(callback=remove_noise, title=title, is_cuda=self.cfg.is_cuda, kwargs=kw) | |
| if _rs: | |
| self.clone_ref = self.cfg.vocal if self.cfg.vocal and Path(self.cfg.vocal).exists() else _remove_noise_wav | |
| self.cfg.source_wav = _remove_noise_wav | |
| self.signal(text='remove noise end') | |
| except Exception as e: | |
| logger.exception(f'降噪失败,跳过 {e}', exc_info=True) | |
| self.signal(text=tr("Speech Recognition to Word Processing")) | |
| raw_subtitles = run_recogn( | |
| recogn_type=self.cfg.recogn_type, | |
| uuid=self.uuid, | |
| model_name=self.cfg.model_name, | |
| audio_file=self.cfg.source_wav,# 必选 16000 采样 | |
| detect_language=self.cfg.detect_language, | |
| cache_folder=self.cfg.cache_folder, | |
| is_cuda=self.cfg.is_cuda, | |
| subtitle_type=self.cfg.subtitle_type, | |
| max_speakers=self.max_speakers, | |
| llm_post=self.cfg.rephrase==1 | |
| ) | |
| if self._exit(): return | |
| if not raw_subtitles: | |
| raise SpeechToTextError(self.cfg.basename + tr('recogn result is empty')) | |
| # 如果是 tiqu,并且不需要翻译,并且需要移除标点 | |
| if self.cfg.app_mode=='tiqu' and not self.should_trans and self.cfg.fix_punc==2: | |
| logger.debug('仅提取不翻译模式下,移除所有标点') | |
| for it in raw_subtitles: | |
| it['text'] = delete_punc(it['text']) | |
| self._save_srt_target(raw_subtitles, self.cfg.source_sub) | |
| self.source_srt_list = raw_subtitles | |
| # 中英恢复标点符号 | |
| if self.cfg.fix_punc==1 and self.cfg.detect_language[:2] in ['zh', 'en']: | |
| down_file_from_ms(f'{ROOT_DIR}/models/puntc', [ | |
| "https://www.modelscope.cn/models/himyworld/videotrans/resolve/master/puntc/model.onnx", | |
| "https://www.modelscope.cn/models/himyworld/videotrans/resolve/master/puntc/config.yaml", | |
| "https://www.modelscope.cn/models/himyworld/videotrans/resolve/master/puntc/tokens.json", | |
| ], callback=self._process_callback) | |
| from videotrans.process.prepare_audio import fix_punc | |
| # 预先删掉已有的标点 | |
| text_dict = {f'{it["line"]}': re.sub(r'[,.?!,。?!]', ' ', it["text"]) for it in self.source_srt_list} | |
| # 序列化后传递文件路径 | |
| text_dict_file=f'{self.cfg.cache_folder}/text_dict_file_{time.time()}.json' | |
| Path(text_dict_file).write_text(json.dumps(text_dict),encoding="utf-8") | |
| kw = {"text_dict_file": text_dict_file, "is_cuda": self.cfg.is_cuda} | |
| try: | |
| _rs = self._new_process(callback=fix_punc, title=tr("Restoring punct"), is_cuda=self.cfg.is_cuda, | |
| kwargs=kw) | |
| if _rs: | |
| text_dict_obj=json.loads(Path(text_dict_file).read_text(encoding='utf-8')) | |
| for it in self.source_srt_list: | |
| it['text'] = text_dict_obj.get(f'{it["line"]}', it['text']) | |
| if self.cfg.detect_language[:2] == 'en': | |
| it['text'] = it['text'].replace(',', ',').replace('。', '. ').replace('?', '?').replace('!','!') | |
| self._save_srt_target(self.source_srt_list, self.cfg.source_sub) | |
| else: | |
| logger.error('标点恢复出错了,跳过') | |
| except Exception as e: | |
| logger.exception(f'标点恢复失败,跳过 {e}', exc_info=True) | |
| self.signal(text=Path(self.cfg.source_sub).read_text(encoding='utf-8'), type='replace_subtitle') | |
| # whisperx-api | |
| # openairecogn并且模型是gpt-4o-transcribe-diarize | |
| # funasr并且模型是paraformer-zh | |
| # deepgram | |
| # 以上这些本身已有说话人识别,如果已有说话人识别结果,就不再重新断句 | |
| if Path(self.cfg.cache_folder + "/speaker.json").exists(): | |
| self._recogn_succeed() | |
| self.signal(text=tr('endtiquzimu')) | |
| return | |
| if self.cfg.rephrase==1: | |
| # LLM重新断句 | |
| try: | |
| from videotrans.translator._openaicompat import OpenAICampat | |
| ob = OpenAICampat( | |
| ainame='chatgpt' if settings.get('llm_ai_type', 'chatgpt') != 'deepseek' else 'deepseek', | |
| uuid=self.uuid) | |
| self.signal(text=tr("Re-segmenting...")) | |
| srt_list = ob.llm_segment(self.source_srt_list ) | |
| if srt_list and len(srt_list) > len(self.source_srt_list) / 2: | |
| self.source_srt_list = srt_list | |
| # 保留原始字幕LLM重新断句之前的文件 | |
| #shutil.copy2(self.cfg.source_sub, f'{self.cfg.source_sub[:-4]}-origin_without_LLM.srt') | |
| self._save_srt_target(self.source_srt_list, self.cfg.source_sub) | |
| else: | |
| logger.error(f'重新断句失败,已恢复原样,原始字幕行:{len(self.source_srt_list)}, 重新断句后字幕行:{len(srt_list)}\n断句结果:\n{srt_list=}') | |
| except Exception as e: | |
| self.signal(text=tr("Re-segmenting Error")) | |
| logger.exception(f"重新断句失败,已恢复原样 {e}", exc_info=True) | |
| self._recogn_succeed() | |
| self.signal(text=tr('endtiquzimu')) | |
| logger.debug(f'[语音识别阶段结束耗时]:{time.time()-_st}s') | |
| def _recogn_succeed(self) -> None: | |
| self.precent += 5 | |
| if self.cfg.app_mode == 'tiqu' and not self.should_trans: | |
| shutil.copy2(self.cfg.source_sub, f"{self.cfg.target_dir}/{self.cfg.noextname}.srt") | |
| self.signal(text=tr('endtiquzimu')) | |
| # 配音后再次对配音文件进行识别,以便生成简短的字幕, | |
| def recogn2pass(self) -> None: | |
| _st=time.time() | |
| if not self.should_recogn2 or self._exit(): | |
| return | |
| if not vail_file(self.cfg.target_wav): | |
| logger.debug(f'跳过二次识别,因无配音音频文件') | |
| return | |
| self.precent += 3 | |
| self.signal(text=tr("Secondary speech recognition of dubbing files")) | |
| shibie_audio = f'{self.cfg.cache_folder}/recogn2pass-{time.time()}.wav' | |
| outsrt_file = f'{self.cfg.cache_folder}/recogn2pass-{time.time()}.srt' | |
| try: | |
| conver_to_16k(self.cfg.target_wav, shibie_audio) | |
| except Exception as e: | |
| logger.exception(f'二次识别配音音频生成字幕时,预处理音频失败,静默跳过 {e}', exc_info=True) | |
| return | |
| if not vail_file(shibie_audio): | |
| logger.error(f'二次识别配音音频生成字幕时,预处理音频失败,静默跳过') | |
| return | |
| try: | |
| # 判断原渠道是否支持目标语言的识别 self.cfg.target_language_code | |
| recogn_type = self.cfg.recogn_type | |
| model_name = self.cfg.model_name | |
| detect_language = self.cfg.target_language_code.split('-')[0] | |
| if recogn_allow_lang(langcode=self.cfg.target_language_code, | |
| recogn_type=recogn_type, | |
| model_name=model_name) is not True: | |
| recogn_type = FASTER_WHISPER | |
| model_name = 'large-v3-turbo' | |
| raw_subtitles = run_recogn( | |
| recogn_type=recogn_type, | |
| uuid=self.uuid, | |
| model_name=model_name, | |
| audio_file=shibie_audio, | |
| detect_language=detect_language, | |
| cache_folder=self.cfg.cache_folder, | |
| is_cuda=self.cfg.is_cuda, | |
| recogn2pass=True # 二次识别 | |
| ) | |
| if self._exit(): return | |
| if not raw_subtitles: | |
| logger.error('二次识别出错:' + tr('recogn result is empty')) | |
| return | |
| # LLM重新断句 start 或者存在 recogn2-llm-resegment.txt 文件,则始终在二次识别后断句,适合 clone 角色时,关闭第一次识别后的重新断句,而单独启用二次识别后的断句,因第一次识别后如果重新断句会导致参考音频裁切不准确 | |
| if self.cfg.rephrase==1 or Path(f'{ROOT_DIR}/recogn2-llm-resegment.txt').exists(): | |
| try: | |
| from videotrans.translator._openaicompat import OpenAICampat | |
| ob = OpenAICampat( | |
| ainame='chatgpt' if settings.get('llm_ai_type', 'chatgpt') != 'deepseek' else 'deepseek', | |
| uuid=self.uuid) | |
| self.signal(text=tr("Re-segmenting...")) | |
| srt_list = ob.llm_segment(raw_subtitles,step="2") | |
| if srt_list and len(srt_list) > len(raw_subtitles) / 2: | |
| # 目标语言代码-recogn2_without_LLM.srt 配音文件二次识别后生成的字幕,未被LLM重新断句之前的文件 | |
| # 原始语言代码-origin_without_LLM.srt 原始音视频文件语音识别后生成的字幕,未被LLM重新断句之前的文件 | |
| #self._save_srt_target(raw_subtitles, f'{self.cfg.target_sub[:-4]}-recogn2_without_LLM.srt') | |
| raw_subtitles = srt_list | |
| else: | |
| logger.error(f'二次识别后LLM重新断句失败,已恢复原样,原始字幕行:{len(raw_subtitles)}, 重新断句后字幕行:{len(srt_list)}\n断句结果:\n{srt_list=}') | |
| except Exception as e: | |
| self.signal(text=tr("Re-segmenting Error")) | |
| logger.exception(f"二次识别后重新断句失败,已恢复原样 {e}", exc_info=True) | |
| # LLM重新断句 end | |
| if self.cfg.fix_punc==2: | |
| logger.debug('二次识别后,移除所有标点') | |
| for it in raw_subtitles: | |
| it['text']=delete_punc(it['text']) | |
| self._save_srt_target(raw_subtitles, outsrt_file) | |
| if not vail_file(outsrt_file): | |
| logger.error(f'二次识别配音文件失败,原因未知') | |
| return | |
| # 覆盖 | |
| shutil.copy2(outsrt_file, self.cfg.target_sub) | |
| self.signal(text='STT 2 pass end') | |
| logger.debug('二次识别成功完成') | |
| except Exception as e: | |
| logger.exception(f'二次识别配音音频生成字幕时失败,静默跳过 {e}', exc_info=True) | |
| return | |
| logger.debug(f'[二次识别阶段结束耗时]:{time.time()-_st}s') | |
| def diariz(self): | |
| _st=time.time() | |
| # 说话人设为1,不进行分离 | |
| if self._exit() or not self.cfg.enable_diariz or self.max_speakers == 1 or Path( | |
| self.cfg.cache_folder + "/speaker.json").exists(): | |
| return | |
| # built pyannote reverb ali_CAM | |
| speaker_type = settings.get('speaker_type', 'built') | |
| hf_token = settings.get('hf_token') | |
| if speaker_type == 'built' and self.cfg.detect_language[:2] not in ['zh', 'en']: | |
| logger.error(f'当前选择 built 说话人分离模型,但不支持当前语言:{self.cfg.detect_language}') | |
| return | |
| if speaker_type in ['pyannote', 'reverb'] and not hf_token: | |
| logger.error(f'当前选择 pyannote 说话人分离模型,但未设置 huggingface.co 的token: {self.cfg.detect_language}') | |
| return | |
| hf_endpoit = "https://huggingface.co" | |
| if speaker_type in ['pyannote', 'reverb']: | |
| # 判断是否可访问 huggingface.co | |
| try: | |
| import requests | |
| requests.head('https://huggingface.co', timeout=5) | |
| except Exception: | |
| logger.exception(f'当前选择 {speaker_type} 说话人分离模型,但无法连接到 https://huggingface.co,可能会失败', exc_info=True) | |
| hf_endpoit = "https://hf-mirror.com" | |
| from videotrans.util.help_down import down_file_from_ms, check_and_down_ms | |
| try: | |
| self.precent += 3 | |
| title = tr(f'Begin separating the speakers') + f':{speaker_type}' | |
| subtitles_file=f'{self.cfg.cache_folder}/diariz-{time.time()}.json' | |
| Path(subtitles_file).write_text(json.dumps([[it['start_time'], it['end_time']] for it in self.source_srt_list]),encoding='utf-8') | |
| kw = { | |
| "input_file": self.cfg.source_wav, | |
| "subtitles_file": subtitles_file, | |
| "speak_file":self.cfg.cache_folder + "/speaker.json", | |
| "num_speakers": self.max_speakers, | |
| "is_cuda": self.cfg.is_cuda | |
| } | |
| if speaker_type == 'built': | |
| down_file_from_ms(f'{ROOT_DIR}/models/onnx', [ | |
| "https://www.modelscope.cn/models/himyworld/videotrans/resolve/master/onnx/seg_model.onnx", | |
| "https://www.modelscope.cn/models/himyworld/videotrans/resolve/master/onnx/nemo_en_titanet_small.onnx", | |
| "https://www.modelscope.cn/models/himyworld/videotrans/resolve/master/onnx/3dspeaker_speech_eres2net_large_sv_zh-cn_3dspeaker_16k.onnx" | |
| ], callback=self._process_callback) | |
| from videotrans.process.prepare_audio import built_speakers as _run_speakers | |
| del kw['is_cuda'] | |
| kw['num_speakers'] = -1 if self.max_speakers < 1 else self.max_speakers | |
| kw['language'] = self.cfg.detect_language | |
| elif speaker_type == 'ali_CAM': | |
| check_and_down_ms(model_id='iic/speech_campplus_speaker-diarization_common', | |
| callback=self._process_callback) | |
| from videotrans.process.prepare_audio import cam_speakers as _run_speakers | |
| elif speaker_type == 'pyannote': | |
| from videotrans.process.prepare_audio import pyannote_speakers as _run_speakers | |
| elif speaker_type == 'reverb': | |
| from videotrans.process.prepare_audio import reverb_speakers as _run_speakers | |
| else: | |
| logger.error(f'当前所选说话人分离模型不支持:{speaker_type=}') | |
| return | |
| if speaker_type in ['pyannote', 'reverb']: | |
| self.signal(text='Downloading speakers models') | |
| from huggingface_hub import snapshot_download | |
| snapshot_download( | |
| repo_id="pyannote/speaker-diarization-3.1" if speaker_type == 'pyannote' else "Revai/reverb-diarization-v1", | |
| token=hf_token, | |
| endpoint=hf_endpoit | |
| ) | |
| _rs = self._new_process(callback=_run_speakers, title=title, | |
| is_cuda=self.cfg.is_cuda and speaker_type != 'built', kwargs=kw) | |
| if _rs: | |
| logger.debug('分离说话人成功完成') | |
| shutil.copy2(self.cfg.cache_folder + "/speaker.json", self.cfg.target_dir + "/speaker.json") | |
| else: | |
| logger.error('分离失败说话人失败') | |
| self.signal(text=tr('separating speakers end')) | |
| except Exception as e: | |
| logger.exception(f'说话人分离失败,跳过 {e}', exc_info=True) | |
| logger.debug(f'[说话人分离阶段结束耗时]:{time.time()-_st}s') | |
| # 翻译字幕文件 | |
| def trans(self) -> None: | |
| _st=time.time() | |
| if self._exit() or not self.should_trans: return | |
| self.precent += 3 | |
| self.signal(text=tr('starttrans')) | |
| # 如果存在目标语言字幕,无需继续翻译,前台直接使用该字幕替换 | |
| if vail_file(self.cfg.target_sub): | |
| self.signal( | |
| text=Path(self.cfg.target_sub).read_text(encoding="utf-8", errors="ignore"), | |
| type='replace_subtitle' | |
| ) | |
| return | |
| rawsrt = get_subtitle_from_srt(self.cfg.source_sub, is_file=True) | |
| self.signal(text=tr('kaishitiquhefanyi')) | |
| target_srt = run_trans( | |
| translate_type=self.cfg.translate_type, | |
| text_list=copy.deepcopy(rawsrt), | |
| uuid=self.uuid, | |
| source_code=self.cfg.source_language_code, | |
| target_code=self.cfg.target_language_code | |
| ) | |
| if self._exit(): return | |
| # 一一核对每条字幕,翻译可能导致每条字幕开头结尾出现3个 . 符号,配音后和无需配音时,需清理 | |
| target_srt = self.check_target_sub(rawsrt, target_srt) | |
| if not self.should_dubbing: | |
| for it in target_srt: | |
| it['text']=it['text'].strip('...') | |
| # 如果仅提取 | |
| if self.cfg.app_mode=='tiqu': | |
| # 移除标点 | |
| if self.cfg.fix_punc==2: | |
| logger.debug('仅提取模式下,移除所有标点') | |
| for it in rawsrt: | |
| it['text']=delete_punc(it['text']) | |
| for it in target_srt: | |
| it['text']=delete_punc(it['text']) | |
| # 保存翻译前的字幕 | |
| self._save_srt_target(rawsrt, f"{self.cfg.target_dir}/{self.cfg.noextname}-{self.cfg.source_language_code}.srt") | |
| # 双语输出 | |
| if self.cfg.output_srt > 0 and self.cfg.source_language_code != self.cfg.target_language_code: | |
| _source_srt_len = len(rawsrt) | |
| for i, it in enumerate(target_srt): | |
| if i < _source_srt_len and self.cfg.output_srt == 1: | |
| # 目标语言在下 | |
| it['text'] = ("\n".join([rawsrt[i]['text'].strip(), it['text'].strip()])).strip() | |
| elif i < _source_srt_len and self.cfg.output_srt == 2: | |
| it['text'] = ("\n".join([it['text'].strip(), rawsrt[i]['text'].strip()])).strip() | |
| # 翻译后的字幕 | |
| self._save_srt_target(target_srt, self.cfg.target_sub) | |
| if self.cfg.app_mode == 'tiqu': | |
| _output_file = f"{self.cfg.target_dir}/{self.cfg.noextname}.srt" | |
| if self.cfg.copysrt_rawvideo: | |
| p = Path(self.cfg.name) | |
| _output_file = f'{p.parent.as_posix()}/{p.stem}.srt' | |
| if not Path(_output_file).exists(): | |
| shutil.copy2(self.cfg.target_sub, _output_file) | |
| self.signal(text=tr('endtrans')) | |
| logger.debug(f'[字幕翻译阶段结束耗时]:{time.time()-_st}s') | |
| # 对字幕进行配音 | |
| def dubbing(self) -> None: | |
| _st=time.time() | |
| if self._exit() or self.cfg.app_mode == 'tiqu': | |
| return | |
| if self.should_dubbing: | |
| self.signal(text=tr('kaishipeiyin')) | |
| self.precent += 3 | |
| # 内部判断,如果不需要配音则直接跳过,然后进行后续移除标点和3个点操作,不可本方法开头跳过 | |
| self._tts() | |
| # 配音完毕后,需更新 目标字幕,移除前后3个点 | |
| if Path(self.cfg.target_sub).exists(): | |
| subs = get_subtitle_from_srt(self.cfg.target_sub) | |
| if self.cfg.fix_punc==2: | |
| logger.debug('配音结束后,移除目标字幕中所有标点') | |
| for it in subs: | |
| it['text']=it['text'].strip('...') | |
| if self.cfg.fix_punc==2: | |
| it['text']=delete_punc(it['text']) | |
| if self.cfg.fix_punc==2 and Path(self.cfg.source_sub).exists(): | |
| logger.debug('配音结束后,移除原始字幕中所有标点') | |
| subs = get_subtitle_from_srt(self.cfg.source_sub) | |
| for it in subs: | |
| it['text']=delete_punc(it['text']) | |
| self._save_srt_target(subs, self.cfg.source_sub) | |
| if self.should_dubbing: | |
| self.signal(text=tr('The dubbing is finished')) | |
| logger.debug(f'[语音合成阶段结束耗时]:{time.time()-_st}s') | |
| # 音画字幕对齐 | |
| def align(self) -> None: | |
| _st=time.time() | |
| if self._exit() or self.cfg.app_mode == 'tiqu' or not self.should_dubbing or self.ignore_align: | |
| return | |
| self.signal(text=tr('duiqicaozuo')) | |
| self.precent += 3 | |
| if self.cfg.voice_autorate or self.cfg.video_autorate: | |
| self.signal(text=tr("Sound & video speed alignment stage")) | |
| # 需要视频慢速,则判断无声视频是否已分离完毕 | |
| if self.cfg.video_autorate: | |
| is_novoice_mp4(self.cfg.novoice_mp4, self.uuid) | |
| # 存在视频,则以视频长度为准 | |
| if vail_file(self.cfg.novoice_mp4): | |
| self.video_time = get_video_duration(self.cfg.novoice_mp4) | |
| from videotrans.task._rate import SpeedRate | |
| rate_inst = SpeedRate( | |
| queue_tts=self.queue_tts, | |
| uuid=self.uuid, | |
| should_audiorate=self.cfg.voice_autorate, | |
| # 视频是否需慢速,需要时对 novoice_mp4进行处理 | |
| should_videorate=self.cfg.video_autorate if not self.is_audio_trans else False, | |
| novoice_mp4=self.cfg.novoice_mp4 if not self.is_audio_trans else None, | |
| # 原始总时长 | |
| raw_total_time=self.video_time, | |
| target_audio=self.cfg.target_wav, | |
| cache_folder=self.cfg.cache_folder, | |
| align_sub_audio=self.cfg.align_sub_audio, # 均在未启用音频加速和视频慢速时才起作用 | |
| remove_silent_mid=self.cfg.remove_silent_mid # 均在未启用音频加速和视频慢速时才起作用 | |
| ) | |
| self.queue_tts = rate_inst.run() | |
| # 慢速处理后,更新新视频总时长,用于音视频对齐 | |
| if vail_file(self.cfg.novoice_mp4): | |
| self.video_time = get_video_duration(self.cfg.novoice_mp4) | |
| # 对齐字幕 | |
| if self.cfg.voice_autorate or self.cfg.video_autorate or self.cfg.align_sub_audio: | |
| srt = "" | |
| for (idx, it) in enumerate(self.queue_tts): | |
| startraw = ms_to_time_string(ms=it['start_time']) | |
| endraw = ms_to_time_string(ms=it['end_time']) | |
| srt += f"{idx + 1}\n{startraw} --> {endraw}\n{it['text'].strip('...')}\n\n" | |
| # 字幕保存到目标文件夹 | |
| with Path(self.cfg.target_sub).open('w', encoding="utf-8") as f: | |
| f.write(srt.strip()) | |
| # 成功后,如果存在 音量,则调节音量 | |
| if self.cfg.tts_type not in [EDGE_TTS, AZURE_TTS] and self.cfg.volume != '+0%' and vail_file( | |
| self.cfg.target_wav): | |
| volume = self.cfg.volume.replace('%', '').strip() | |
| try: | |
| volume = 1 + float(volume) / 100 | |
| if volume != 1.0: | |
| tmp_name = self.cfg.cache_folder + f'/volume-{volume}-{Path(self.cfg.target_wav).name}' | |
| runffmpeg(['-y', '-i', os.path.basename(self.cfg.target_wav), '-af', f"volume={volume}", | |
| os.path.basename(tmp_name)], cmd_dir=self.cfg.cache_folder) | |
| shutil.copy2(tmp_name, self.cfg.target_wav) | |
| except Exception as e: | |
| logger.exception(f'配音后调节音量失败,静默跳过 {e}', exc_info=True) | |
| self.signal(text=tr('Alignment phase complete, awaiting the next step')) | |
| logger.debug(f'[声画字幕对齐阶段结束耗时]:{time.time()-_st}s') | |
| # 将 视频、音频、字幕合成 | |
| def assembling(self) -> None: | |
| _st=time.time() | |
| if self._exit() or self.is_audio_trans or self.cfg.app_mode == 'tiqu' or not self.should_hebing: | |
| return | |
| self.precent = self.precent + 3 if self.precent < 95 else self.precent | |
| self.signal(text=tr('kaishihebing')) | |
| self._join_video_audio_srt() | |
| logger.debug(f'[音频+字幕+画面合成阶段结束耗时]:{time.time()-_st}s') | |
| def task_done(self) -> None: | |
| if self._exit(): return | |
| self.precent = max(99, self.precent) | |
| # 提取时,删除 | |
| if self.cfg.app_mode == 'tiqu': | |
| try: | |
| Path(f"{self.cfg.target_dir}/{self.cfg.source_language_code}.srt").unlink( | |
| missing_ok=True) | |
| Path(f"{self.cfg.target_dir}/{self.cfg.target_language_code}.srt").unlink( | |
| missing_ok=True) | |
| except OSError: | |
| logger.warning('仅提取模式时,清理中间文件失败,跳过') | |
| return self.set_end(True) | |
| if self.is_audio_trans and vail_file(self.cfg.target_wav): | |
| try: | |
| shutil.copy2(self.cfg.target_wav, | |
| f"{self.cfg.target_dir}/{self.cfg.target_language_code}-{self.cfg.noextname}.wav") | |
| except shutil.SameFileError: | |
| pass | |
| try: | |
| if self.cfg.only_out_mp4: | |
| shutil.move(self.cfg.targetdir_mp4, Path(self.cfg.target_dir).parent / Path(self.cfg.targetdir_mp4).name) | |
| shutil.rmtree(self.cfg.target_dir, ignore_errors=True) | |
| except OSError as e: | |
| logger.exception(f'仅输出mp4时清理临时文件移动视频位置出错,跳过 {e}', exc_info=True) | |
| self.set_end(True) | |
| logger.debug(f'[{self.cfg.name}视频翻译任务结束,总耗时]:{time.time()-self.cost_duration}s') | |
| # 从原始视频分离出 无声视频 | |
| def _split_novoice_byraw(self): | |
| cmd = [ | |
| "-y", | |
| "-fflags", | |
| "+genpts", | |
| "-i", | |
| self.cfg.name, | |
| "-an", | |
| "-c:v", | |
| "copy" if self.is_copy_video else f"libx264" | |
| ] | |
| _name = os.path.basename(self.cfg.novoice_mp4) | |
| enc_qua = [] if self.is_copy_video else ['-crf', '18'] | |
| if self.is_copy_video or settings.get('force_lib'): | |
| return runffmpeg(cmd + enc_qua + [_name], noextname=self.uuid, cmd_dir=self.cfg.cache_folder) | |
| try: | |
| hw_decode_args, _, vcodec, enc_args = self._get_hard_cfg(codec="264") | |
| cmd = [ | |
| "-y", | |
| "-fflags", | |
| "+genpts", | |
| ] | |
| cmd += hw_decode_args | |
| cmd += [ | |
| "-i", | |
| self.cfg.name, | |
| "-an", | |
| "-c:v", | |
| vcodec, | |
| _name | |
| ] | |
| self._subprocess(cmd) | |
| app_cfg.queue_novice[self.uuid] = 'end' | |
| except Exception as e: | |
| logger.exception(f'硬件分离无声视频失败,尝试软分离 {e}', exc_info=True) | |
| return runffmpeg([ | |
| "-y", | |
| "-fflags", | |
| "+genpts", | |
| "-i", | |
| self.cfg.name, | |
| "-an", | |
| "-c:v", | |
| "libx264", | |
| _name | |
| ], noextname=self.uuid, cmd_dir=self.cfg.cache_folder, force_cpu=True) | |
| # 从原始视频中分离出音频 | |
| def _split_audio_byraw(self, is_separate=False): | |
| cmd = [ | |
| "-y", | |
| "-i", | |
| self.cfg.name, | |
| "-vn", | |
| "-ac", | |
| "1", | |
| "-ar", | |
| "16000", | |
| "-c:a", | |
| "pcm_s16le", | |
| self.cfg.source_wav | |
| ] | |
| rs = runffmpeg(cmd) | |
| if not is_separate: | |
| return rs | |
| # 继续人声分离 | |
| tmpfile = self.cfg.cache_folder + "/441000_ac2_raw.wav" | |
| runffmpeg([ | |
| "-y", | |
| "-i", | |
| self.cfg.name, | |
| "-vn", | |
| "-ac", | |
| "2", | |
| "-ar", | |
| "44100", | |
| "-c:a", | |
| "pcm_s16le", | |
| tmpfile | |
| ]) | |
| if vail_file(self.cfg.vocal) and vail_file(self.cfg.instrument): | |
| return | |
| from videotrans.util.help_down import down_file_from_ms | |
| title = tr('Separating vocals and background music, which may take a longer time') | |
| uvr_models = settings.get('uvr_models') | |
| if uvr_models.startswith('spleeter'): | |
| down_file_from_ms(f'{ROOT_DIR}/models/onnx', [ | |
| f"https://www.modelscope.cn/models/himyworld/videotrans/resolve/master/onnx/vocals.fp16.onnx", | |
| f"https://www.modelscope.cn/models/himyworld/videotrans/resolve/master/onnx/accompaniment.fp16.onnx" | |
| ], callback=self._process_callback) | |
| else: | |
| down_file_from_ms(f'{ROOT_DIR}/models/onnx', [ | |
| f"https://www.modelscope.cn/models/himyworld/videotrans/resolve/master/onnx/{uvr_models}.onnx" | |
| ], callback=self._process_callback) | |
| from videotrans.process.prepare_audio import vocal_bgm | |
| # 返回 False None 失败 | |
| kw = {"input_file": tmpfile, "vocal_file": self.cfg.vocal, "instr_file": self.cfg.instrument, | |
| "uvr_models": uvr_models} | |
| try: | |
| rs = self._new_process(callback=vocal_bgm, title=title, is_cuda=False, kwargs=kw) | |
| if rs and vail_file(self.cfg.vocal) and vail_file(self.cfg.instrument): | |
| cmd = [ | |
| "-y", | |
| "-i", | |
| self.cfg.vocal, | |
| "-ac", | |
| "1", | |
| "-ar", | |
| "16000", | |
| "-c:a", | |
| "pcm_s16le", | |
| '-af', | |
| "volume=1.5", | |
| self.cfg.source_wav | |
| ] | |
| runffmpeg(cmd) | |
| shutil.copy2(self.cfg.vocal, f'{self.cfg.target_dir}/vocal.wav') | |
| shutil.copy2(self.cfg.instrument, f'{self.cfg.target_dir}/instrument.wav') | |
| except Exception as e: | |
| logger.exception(f'人声背景声分离失败,静默跳过 {e}', exc_info=True) | |
| # 配音预处理,去掉无效字符,整理开始时间 | |
| def _tts(self) -> None: | |
| if not self.should_dubbing: | |
| self.signal(text='Skip tts') | |
| return | |
| queue_tts = [] | |
| subs = get_subtitle_from_srt(self.cfg.target_sub) | |
| source_subs = get_subtitle_from_srt(self.cfg.source_sub) | |
| if len(subs) < 1: | |
| raise DubbingSrtError(f"SRT file error:{self.cfg.target_sub}") | |
| try: | |
| rate = int(str(self.cfg.voice_rate).replace('%', '')) | |
| except (ValueError,TypeError): | |
| rate = 0 | |
| rate = f"+{rate}%" if rate >= 0 else f"{rate}%" | |
| # 取出设置的每行角色 | |
| line_roles = app_cfg.line_roles | |
| voice_role = self.cfg.voice_role | |
| #force_clone = str(voice_role).strip().lower() == 'clone' and self.cfg.tts_type in SUPPORT_CLONE | |
| logger.debug(f'{line_roles=}') | |
| # 取出每一条字幕,行号\n开始时间 --> 结束时间\n内容 | |
| for i, it in enumerate(subs): | |
| if it['end_time'] < it['start_time'] or not it['text'].strip(): | |
| continue | |
| # 判断是否存在单独设置的行角色,如果不存在则使用全局 | |
| #voice = 'clone' if force_clone else line_roles.get(f'{it["line"]}', voice_role) | |
| voice = line_roles.get(f'{it["line"]}', voice_role) if line_roles else voice_role | |
| _key = get_md5(f"{self.cfg.target_language_code}-{it['text']}-{voice}-{rate}-{self.cfg.volume}-{self.cfg.pitch}-{self.cfg.tts_type}") | |
| tmp_dict = { | |
| "text": it['text'], | |
| "line": it['line'], | |
| "start_time": it['start_time'], | |
| "end_time": it['end_time'], | |
| "startraw": it['startraw'], | |
| "endraw": it['endraw'], | |
| "ref_text": source_subs[i]['text'] if source_subs and i < len(source_subs) else '', | |
| "start_time_source": source_subs[i]['start_time'] if source_subs and i < len(source_subs) else it[ | |
| 'start_time'], | |
| "end_time_source": source_subs[i]['end_time'] if source_subs and i < len(source_subs) else it[ | |
| 'end_time'], | |
| "role": voice, | |
| "rate": rate, | |
| "volume": self.cfg.volume, | |
| "pitch": self.cfg.pitch, | |
| "tts_type": self.cfg.tts_type, | |
| "filename": f"{self.cfg.cache_folder}/dubb-{i}-{_key}.wav" | |
| } | |
| # 如果是 clone 角色, 需要截取对应片段 | |
| if str(voice).strip().lower() == 'clone' and self.cfg.tts_type in SUPPORT_CLONE: | |
| tmp_dict['ref_wav'] = f"{self.cfg.cache_folder}/clone-{i}.wav" | |
| tmp_dict['ref_language'] = self.cfg.detect_language[:2] | |
| queue_tts.append(tmp_dict) | |
| self.queue_tts = copy.deepcopy(queue_tts) | |
| if not self.queue_tts or len(self.queue_tts) < 1: | |
| raise RuntimeError(f'字幕长度为0,无法继续配音') | |
| # 如果存在有 ref_wav 即需要clone,存在参考音频的 | |
| if len([it.get("ref_wav") for it in self.queue_tts if it.get("ref_wav")]) > 0: | |
| self._create_ref_from_vocal() | |
| # 调用配音渠道 | |
| run_tts( | |
| queue_tts=self.queue_tts, | |
| language=self.cfg.target_language_code, | |
| uuid=self.uuid, | |
| tts_type=self.cfg.tts_type, | |
| is_cuda=self.cfg.is_cuda | |
| ) | |
| # 为每条字幕保留原始配音片段 | |
| if settings.get('save_segment_audio', False): | |
| outname = self.cfg.target_dir + f'/segment_audio_{self.cfg.noextname}' | |
| Path(outname).mkdir(parents=True, exist_ok=True) | |
| for it in self.queue_tts: | |
| text = re.sub(r'["\'*?\\/|:<>\r\n\t]+', '', it['text'], flags=re.I | re.S) | |
| name = f'{outname}/{it["line"]}-{text[:60]}.wav' | |
| if Path(it['filename']).exists(): | |
| shutil.copy2(it['filename'], name) | |
| # 多线程实现裁剪参考音频 | |
| def _create_ref_from_vocal(self): | |
| # 保底原始音频用于克隆时参考音频 | |
| vocal = self.cfg.source_wav | |
| if self.clone_ref and Path(self.clone_ref).exists(): | |
| # 人声背景声分离 出来的人声音频,44.1k,如果有降噪,则为 16000 | |
| vocal=self.clone_ref | |
| else: | |
| # 无则从原始视频中提取44.1k音频作为参考音频 | |
| try: | |
| tmpfile = self.cfg.cache_folder + "/clone_ref_44100.wav" | |
| runffmpeg([ | |
| "-y", | |
| "-i", | |
| self.cfg.name, | |
| "-vn", | |
| "-ac", | |
| "1", | |
| "-ar", | |
| "44100", | |
| "-c:a", | |
| "pcm_s16le", | |
| tmpfile | |
| ]) | |
| vocal=tmpfile | |
| except Exception as e: | |
| logger.exception(f'克隆语音前分离出 44.1k 的原始音频失败',exc_info=True) | |
| logger.debug(f'语音克隆模式下,所用参考音频为:{vocal}') | |
| # 裁切对应片段为参考音频 | |
| def _cutaudio_from_vocal(it): | |
| try: | |
| logger.debug(f"裁切对应片段为参考音频:{it['startraw']}->{it['endraw']}\n当前{it=}") | |
| cut_from_audio( | |
| audio_file=vocal, | |
| ss=it['startraw'], | |
| to=it['endraw'], | |
| out_file=it['ref_wav'] | |
| ) | |
| except Exception as e: | |
| logger.exception(f'裁切参考音频失败:{it=},{e}', exc_info=True) | |
| all_task = [] | |
| with ThreadPoolExecutor(max_workers=min(8, len(self.queue_tts), os.cpu_count())) as pool: | |
| for item in self.queue_tts: | |
| if item.get('ref_wav'): | |
| all_task.append(pool.submit(_cutaudio_from_vocal, item)) | |
| if len(all_task) > 0: | |
| _ = [i.result() for i in all_task] | |
| # 添加手动上传的额外背景音乐 | |
| def _back_music(self) -> None: | |
| if self._exit() or not self.should_dubbing or not vail_file(self.cfg.target_wav) or not vail_file( self.cfg.background_music): | |
| return | |
| try: | |
| self.signal(text=tr("Adding background audio")) | |
| # 获取视频长度 | |
| vtime = get_audio_time(self.cfg.target_wav) | |
| # 获取背景音频长度 | |
| atime = get_audio_time(self.cfg.background_music) | |
| if atime < 1: | |
| return | |
| bgm_file = self.cfg.cache_folder + f'/bgm_file.wav' | |
| self.convert_to_wav(self.cfg.background_music, bgm_file) | |
| self.cfg.background_music = bgm_file | |
| beishu = math.ceil(vtime / atime) | |
| if self.cfg.loop_backaudio and beishu > 1 and vtime - 1000 > atime: | |
| # 获取延长片段 | |
| file_list = [self.cfg.background_music for n in range(beishu + 1)] | |
| concat_txt = self.cfg.cache_folder + f'/{time.time()}.txt' | |
| create_concat_txt(file_list, concat_txt=concat_txt) | |
| concat_multi_audio( | |
| concat_txt=concat_txt, | |
| out=self.cfg.cache_folder + "/bgm_file_extend.wav") | |
| self.cfg.background_music = self.cfg.cache_folder + "/bgm_file_extend.wav" | |
| # 背景音频和配音合并 | |
| cmd = ['-y', | |
| '-i', os.path.basename(self.cfg.target_wav), | |
| '-i', self.cfg.background_music, | |
| '-filter_complex', "[0:a][1:a]amix=inputs=2:duration=first:dropout_transition=2", | |
| '-ac', '2', | |
| '-c:a', 'pcm_s16le', | |
| "lastend.wav" | |
| ] | |
| runffmpeg(cmd, cmd_dir=self.cfg.cache_folder) | |
| self.cfg.target_wav = self.cfg.cache_folder + f"/lastend.wav" | |
| except Exception as e: | |
| logger.exception(f'添加背景音乐失败,静默跳过 {e}', exc_info=True) | |
| # 重新嵌回分离出的背景声音 | |
| def _separate(self) -> None: | |
| # 如果背景音频分离失败,则静默返回 | |
| if self._exit() or not self.cfg.embed_bgm or not vail_file(self.cfg.instrument) or not vail_file(self.cfg.target_wav): | |
| return | |
| try: | |
| self.signal(text=tr("Re-embedded background sounds")) | |
| vtime = get_audio_time(self.cfg.target_wav) | |
| atime = get_audio_time(self.cfg.instrument) | |
| if atime < 1: | |
| return | |
| beishu = math.ceil(vtime / atime) | |
| instrument_file = self.cfg.instrument | |
| logger.debug(f'合并背景音 {beishu=},{atime=},{vtime=}') | |
| if atime + 1000 < vtime: | |
| if int(self.cfg.loop_backaudio) == 1: | |
| # 背景音通过循环方式延长 | |
| file_list = [instrument_file for n in range(beishu + 1)] | |
| concat_txt = self.cfg.cache_folder + f'/{time.time()}.txt' | |
| create_concat_txt(file_list, concat_txt=concat_txt) | |
| concat_multi_audio(concat_txt=concat_txt, | |
| out=self.cfg.cache_folder + "/instrument-concat.wav") | |
| else: | |
| # 通过变速延长背景音 | |
| change_speed_rubberband(instrument_file, self.cfg.cache_folder + f"/instrument-concat.wav", | |
| vtime) | |
| instrument_file = self.cfg.cache_folder + f"/instrument-concat.wav" | |
| tmp_out_wav = Path(self.cfg.cache_folder + f'/{time.time()}-1.wav').as_posix() | |
| tmp_volume = Path(self.cfg.cache_folder + f'/{time.time()}.wav').as_posix() | |
| # 背景音量降低 | |
| self.convert_to_wav(instrument_file, tmp_volume, ["-filter:a", f"volume={self.cfg.backaudio_volume}"]) | |
| runffmpeg(['-y', '-i', os.path.basename(self.cfg.target_wav), '-i', os.path.basename(tmp_volume), | |
| '-filter_complex', | |
| "[0:a][1:a]amix=inputs=2:duration=first:dropout_transition=2", '-ac', '2', "-b:a", "128k", | |
| '-c:a', 'pcm_s16le', os.path.basename(tmp_out_wav)], cmd_dir=self.cfg.cache_folder) | |
| shutil.copy2(tmp_out_wav, self.cfg.target_wav) | |
| except Exception as e: | |
| logger.exception(f'重新嵌入分离的背景音失败 {e}', exc_info=True) | |
| # 处理所需字幕 | |
| def _process_subtitles(self) -> Union[tuple[str, str], None]: | |
| logger.debug(f"\n======准备要嵌入的字幕:{self.cfg.subtitle_type=}=====") | |
| if not Path(self.cfg.target_sub).exists() : | |
| raise VideoTransError(tr("No valid subtitle file exists")+self.cfg.target_sub) | |
| # 如果原始语言和目标语言相同,或不存原始语言字幕,则强制单字幕 | |
| if not Path(self.cfg.source_sub).exists() or (self.cfg.source_language_code == self.cfg.target_language_code): | |
| if self.cfg.subtitle_type == 3: | |
| self.cfg.subtitle_type = 1 | |
| elif self.cfg.subtitle_type == 4: | |
| self.cfg.subtitle_type = 2 | |
| process_end_subtitle = self.cfg.cache_folder + f'/end.srt' | |
| # 单行字符数 | |
| maxlen = int( | |
| settings.get('cjk_len', 15) if self.cfg.target_language_code[:2] in contants.CJK_LANG else | |
| settings.get('other_len', 60)) | |
| target_sub_list = get_subtitle_from_srt(self.cfg.target_sub) | |
| srt_string = "" | |
| # 双硬字幕时的两种语言字幕分割符,用于定义不同样式 | |
| _join_flag = '' | |
| # 双硬 双软字幕组装 | |
| if self.cfg.subtitle_type in [3, 4]: | |
| source_sub_list = get_subtitle_from_srt(self.cfg.source_sub) | |
| source_length = len(source_sub_list) | |
| # 原语言单行字符长度 | |
| source_maxlen = int( | |
| settings.get('cjk_len', 15) if self.cfg.source_language_code[:2] in ["zh", "ja", "jp", "ko", | |
| 'yu'] else | |
| settings.get('other_len', 60)) | |
| # 双语字幕 | |
| # 判断 双硬字幕 and 存在 ass.json 文件 and (Bottom_Fontsize != Fontsize or PrimaryColour!=Bottom_PrimaryColour) 需要 对双语字幕的第2行设置不同颜色和尺寸 | |
| _join_flag = self._get_join_flag() | |
| for i, it in enumerate(target_sub_list): | |
| # 换行 | |
| _text = simple_wrap(it['text'].strip(), maxlen, self.cfg.target_language_code) | |
| srt_string += f"{it['line']}\n{it['time']}\n" | |
| if source_length > 0 and i < source_length: | |
| _text_source = simple_wrap(source_sub_list[i]['text'], source_maxlen, | |
| self.cfg.source_language_code) | |
| _text = f'{_text_source}\n{_join_flag}{_text}' if self.cfg.output_srt == 1 else f'{_text}\n{_join_flag}{_text_source}' | |
| srt_string += f"{_text}\n\n" | |
| srt_string = srt_string.strip() | |
| process_end_subtitle = f"{self.cfg.cache_folder}/shuang.srt" | |
| Path(process_end_subtitle).write_text(srt_string, encoding='utf-8') | |
| Path(self.cfg.target_dir + "/shuang.srt").write_text( | |
| srt_string.replace('###', '') if _join_flag == '###' else srt_string, encoding='utf-8') | |
| else: | |
| # 单字幕,需处理字符数换行 | |
| for i, it in enumerate(target_sub_list): | |
| tmp = simple_wrap(it['text'].strip(), maxlen, self.cfg.target_language_code) | |
| srt_string += f"{it['line']}\n{it['time']}\n{tmp.strip()}\n\n" | |
| with Path(process_end_subtitle).open('w', encoding='utf-8') as f: | |
| f.write(srt_string) | |
| # 目标字幕语言 | |
| subtitle_langcode = translator.get_subtitle_code(show_target=self.cfg.target_language) | |
| logger.debug( | |
| f'最终确定字幕嵌入类型:{self.cfg.subtitle_type} ,目标字幕语言:{subtitle_langcode}, 字幕文件:{process_end_subtitle}\n') | |
| # 单软 或双软 | |
| if self.cfg.subtitle_type in [2, 4]: | |
| return os.path.basename(process_end_subtitle), subtitle_langcode | |
| # 硬字幕转为ass格式 并设置样式 | |
| process_end_subtitle_ass = set_ass_font(process_end_subtitle) | |
| basename = os.path.basename(process_end_subtitle_ass) | |
| return basename, subtitle_langcode | |
| def _get_join_flag(self): | |
| _join_flag = "" | |
| if self.cfg.subtitle_type != 3 or not Path(f'{ROOT_DIR}/videotrans/ass.json').exists(): | |
| return _join_flag | |
| try: | |
| assjson = json.loads(Path(f'{ROOT_DIR}/videotrans/ass.json').read_text(encoding='utf-8')) | |
| except Exception: | |
| logger.warning(f'未自定义样式 ass.json ,忽略') | |
| return _join_flag | |
| else: | |
| for k, v in assjson.items(): | |
| if k.startswith('Bottom_') and v != assjson.get(k[7:]): | |
| _join_flag = '###' | |
| break | |
| return _join_flag | |
| # 视频定格最后一帧延长末端 | |
| def _video_extend(self, duration_ms=1000): | |
| sec = duration_ms / 1000.0 | |
| final_video_path = Path(f'{self.cfg.cache_folder}/final_video_with_freeze_lastend.mp4').as_posix() | |
| cmd = ['-y', '-i', os.path.basename(self.cfg.novoice_mp4), | |
| '-vf', f'tpad=stop_mode=clone:stop_duration={sec:.3f}', | |
| '-c:v', 'libx264', | |
| '-crf', f'{settings.get("crf", 23)}', | |
| '-preset', settings.get('preset', 'veryfast'), | |
| '-an', 'final_video_with_freeze_lastend.mp4' | |
| ] | |
| try: | |
| runffmpeg(cmd, force_cpu=True, cmd_dir=self.cfg.cache_folder) | |
| if Path(final_video_path).exists(): | |
| shutil.copy2(final_video_path, self.cfg.novoice_mp4) | |
| logger.debug(f"视频定格应延长{duration_ms}ms,实际向上取整秒延长{sec}s,操作成功。") | |
| except Exception as e: | |
| logger.exception(f"视频定格延长操作失败,跳过 {e}", exc_info=True) | |
| # 最终合成视频 | |
| # 各个播放器对长短不一的音视频流处理态度截然不同 | |
| # 为了保证任何平台都能100% 正常播放,先对音频或视频末尾做添加静音或定格预先对齐,再用 -shortest 明确砍齐 | |
| # 无损输出时不对齐流,直接 -shortest 砍掉,防止重新编码导致有损 | |
| def _join_video_audio_srt(self) -> None: | |
| if self._exit() or not self.should_hebing: | |
| return | |
| # 判断 novoice_mp4 是否完成 | |
| is_novoice_mp4(self.cfg.novoice_mp4, self.uuid) | |
| if not Path(self.cfg.novoice_mp4).exists(): | |
| raise VideoTransError(f'{self.cfg.novoice_mp4} 不存在') | |
| # 需要配音但没有配音文件 | |
| if self.should_dubbing and not vail_file(self.cfg.target_wav): | |
| raise VideoTransError(f"{tr('Dubbing')}{tr('anerror')}:{self.cfg.target_wav}") | |
| self.precent = min(max(90, self.precent), 98) | |
| # 最终需嵌入视频的音频,可能是配音后文件,也可能是原始音频(未配音) | |
| target_m4a = self.cfg.cache_folder + "/will_embed.m4a" | |
| # 用于判断输出原始音频是否结束,is True是结束, | |
| output_source_output = True | |
| # 视频时长 | |
| duration_ms = int(get_video_duration(self.cfg.novoice_mp4)) | |
| # 如果视频时长大于音频时长,音频末尾加静音 | |
| if not self.should_dubbing: | |
| # 无配音的使用原始音频 | |
| self._get_origin_audio(target_m4a,duration_ms) | |
| else: | |
| # 单独输出一个高质量 原始音频输出到目标目录,单独线程执行,不影响继续运行 | |
| output_source_output = False | |
| cmd = [ | |
| "-y", | |
| "-i", | |
| self.cfg.name, | |
| "-vn", | |
| "-b:a", "128k", | |
| "-c:a", | |
| "aac", | |
| self.cfg.source_wav_output | |
| ] | |
| def _output(): | |
| nonlocal output_source_output | |
| try: | |
| runffmpeg(cmd) | |
| except Exception as e: | |
| logger.exception(f'单独输出原始视频中音频文件到目标文件夹失败,跳过{e}', exc_info=True) | |
| finally: | |
| output_source_output = True | |
| threading.Thread(target=_output, daemon=True).start() | |
| # 手动添加的背景音乐嵌入 | |
| self._back_music() | |
| # 重新嵌入分离出的背景音 | |
| self._separate() | |
| # 获取音频时长 | |
| audio_ms = get_audio_time(self.cfg.target_wav) | |
| _cmd=[ | |
| "-y", | |
| "-i", | |
| os.path.basename(self.cfg.target_wav) | |
| ] | |
| v_a_offset=duration_ms-audio_ms | |
| # 视频时长大于音频超100ms,加静音 | |
| if v_a_offset>100: | |
| logger.debug(f'视频时长{duration_ms}ms-音频时长{audio_ms}ms={v_a_offset}ms,需延长音频') | |
| _cmd.extend(['-af', f'apad=pad_dur={v_a_offset/1000.0}']) | |
| _cmd.extend([ | |
| "-ac", "2", "-b:a", "128k", "-c:a", "aac", | |
| os.path.basename(target_m4a) | |
| ]) | |
| runffmpeg(_cmd, cmd_dir=self.cfg.cache_folder) | |
| shutil.copy2(target_m4a, self.cfg.target_wav_output) | |
| self.precent = min(max(95, self.precent), 98) | |
| # 输出视频格式 | |
| _video_output_ext = settings.get('out_video_ext', '.mp4') | |
| # 处理所需字幕 | |
| subtitles_file, subtitle_langcode = None, None | |
| if self.cfg.subtitle_type > 0: | |
| subtitles_file, subtitle_langcode = self._process_subtitles() | |
| if _video_output_ext!='.mp4': | |
| subtitle_langcode=translator.get_mkv_code(subtitle_langcode) | |
| # 字幕嵌入时进入视频目录下 | |
| #os.chdir(self.cfg.cache_folder) | |
| # 再次获取处理好末尾的音频真实时长 | |
| audio_ms = get_audio_time(target_m4a) | |
| # 音频时长 - 视频时长差值 >0 则需要定格视频末尾 | |
| a_v_offset=audio_ms-duration_ms | |
| # 如果需要输出的视频是 264 编码,则使用 -c:v copy,因开始和中间编码均为264,可以考虑使用copy (如果无硬字幕嵌入的话) | |
| is_copy_mode = str(self.video_codec_num) == '264' | |
| # 如果原始视频是标准264,需要输出也是264,未视频慢速,未嵌入硬字幕,则放弃视频末尾定格处理,以便实现无损输出 | |
| is_lossless=self.is_copy_video and is_copy_mode and not self.cfg.video_autorate and self.cfg.subtitle_type not in [1, 3] | |
| if is_lossless: | |
| logger.debug(f'当前原始视频是标准264,输出也是264,未视频慢速,未嵌入硬字幕,放弃视频末尾处理,实现无损输出。音频时长-视频时长={a_v_offset}ms'+(',\n音频时长大于视频时长{a_v_offset}ms,理论上视频末尾应定格等待音频播放完毕,但不同播放器可能有不同处理方式,如音频截断,视频末尾黑屏等' if a_v_offset>0 else '')) | |
| # 音频长于视频>500ms才开始末尾定格 | |
| elif a_v_offset > 500: | |
| try: | |
| self._video_extend(a_v_offset) | |
| except Exception as e: | |
| logger.exception(f'定格视频最后一帧时失败,跳过 {e}', exc_info=True) | |
| # 将生成的视频先导出到临时目录,防止包含各种奇怪符号的targetdir_mp4导致ffmpeg失败 | |
| tmp_target_mp4 = self.cfg.cache_folder + f"/laste_target{_video_output_ext}" | |
| self.signal(text=tr("Video + Subtitles + Dubbing in merge")) | |
| try: | |
| protxt = self.cfg.cache_folder + f"/compose{time.time()}.txt" | |
| protxt_basename = os.path.basename(protxt) | |
| threading.Thread(target=self._hebing_pro, args=(protxt,), daemon=True).start() | |
| novoice_mp4_basename = os.path.basename(self.cfg.novoice_mp4) | |
| target_m4a_basename = os.path.basename(target_m4a) | |
| tmp_target_mp4_basename = os.path.basename(tmp_target_mp4) | |
| # 获取可用的硬件 | |
| if not app_cfg.video_codec: | |
| app_cfg.video_codec = get_video_codec() | |
| cmd0 = [ | |
| "-y", | |
| "-progress", | |
| protxt_basename | |
| ] | |
| cmd1 = [ | |
| "-i", | |
| novoice_mp4_basename, | |
| "-i", | |
| target_m4a_basename | |
| ] | |
| enc_qua = ['-crf', f'{settings.get("crf", 23)}', '-preset', settings.get('preset', 'medium')] | |
| # 若选 cfr,无论是否有视频慢速,均固定帧率输出 | |
| # 若选vfr仅在有视频慢速时使用,其他使用ffmpeg默认auto | |
| fps_mode=None | |
| if settings.get('fps_mode')=='cfr': | |
| fps_mode=["-r",f"{self.video_info['video_fps']}","-fps_mode","cfr"] | |
| elif self.cfg.video_autorate: | |
| fps_mode=["-fps_mode","vfr"] | |
| # 无字幕 或 软字幕 | |
| if self.cfg.subtitle_type not in [1, 3]: | |
| # 软字幕 | |
| if self.cfg.subtitle_type in [2, 4]: | |
| cmd1.extend(["-i", subtitles_file]) | |
| cmd1.extend([ | |
| '-map', | |
| '0:v', | |
| '-map', | |
| '1:a' | |
| ]) | |
| if self.cfg.subtitle_type in [2, 4]: | |
| cmd1.extend(['-map', '2:s']) | |
| cmd1.extend([ | |
| "-c:v", | |
| "copy" if is_copy_mode else f"libx{self.video_codec_num}", | |
| "-c:a", | |
| "copy", | |
| ]) | |
| if self.cfg.subtitle_type in [2, 4]: | |
| cmd1.extend([ | |
| "-c:s", | |
| "mov_text" if _video_output_ext == '.mp4' else 'srt', | |
| "-metadata:s:s:0", | |
| f"language={subtitle_langcode}" | |
| ]) | |
| cmd2 = [ | |
| "-movflags", | |
| "+faststart", | |
| ] | |
| if fps_mode: | |
| cmd2.extend(fps_mode) | |
| #"-t", str(duration_s), | |
| cmd2.extend(['-shortest',tmp_target_mp4_basename]) | |
| if is_copy_mode: | |
| logger.debug(f'[最终视频合成]copy模式,无需重新编码:\n{cmd0 + cmd1 + cmd2}') | |
| runffmpeg(cmd0 + cmd1 + cmd2, cmd_dir=self.cfg.cache_folder, force_cpu=True) | |
| elif app_cfg.video_codec.startswith('libx') or settings.get('force_lib'): | |
| # 不支持硬件编码的就无需尝试硬件了 | |
| logger.debug(f'[最终视频合成]不支持硬件编码或指定了强制软编解码:\n{cmd0 + cmd1 + cmd2}') | |
| runffmpeg(cmd0 + cmd1 + enc_qua + cmd2, cmd_dir=self.cfg.cache_folder, force_cpu=True) | |
| else: | |
| # 尝试使用硬件编解码 | |
| hw_decode_args, _, vcodec, enc_args = self._get_hard_cfg() | |
| cmd1[cmd1.index('-c:v') + 1] = vcodec | |
| # 如果硬件处理失败,回退软编 | |
| try: | |
| self._subprocess(cmd0 + hw_decode_args + cmd1 + enc_args + cmd2) | |
| except Exception as e: | |
| cmd1[cmd1.index('-c:v') + 1] = f'libx{self.video_codec_num}' | |
| logger.exception(f'硬件处理视频合成失败,回退软编 {e}', exc_info=True) | |
| runffmpeg(cmd0 + cmd1 + enc_qua + cmd2, cmd_dir=self.cfg.cache_folder, force_cpu=True) | |
| # 硬字幕 | |
| else: | |
| cmd1.append('-filter_complex') | |
| subtitle_filter = [f"[0:v]subtitles=filename='{subtitles_file}'[v_out]"] | |
| cmd2 = [ | |
| "-map", | |
| "[v_out]", | |
| "-map", | |
| "1:a", | |
| "-c:v", | |
| f'libx{self.video_codec_num}', | |
| '-c:a', | |
| 'copy', | |
| ] | |
| cmd3 = ["-movflags", "+faststart"] | |
| if fps_mode: | |
| cmd3.extend(fps_mode) | |
| #"-t", str(duration_s), | |
| cmd3.extend(['-shortest', tmp_target_mp4_basename]) | |
| if app_cfg.video_codec.startswith('libx') or settings.get('force_lib'): | |
| logger.debug(f'[最终视频合成]不支持硬件编解码或指定了强制软编解码:\n{cmd0 + cmd1 + cmd2}') | |
| runffmpeg(cmd0 + cmd1 + subtitle_filter + cmd2 + enc_qua + cmd3, | |
| cmd_dir=self.cfg.cache_folder, force_cpu=True) | |
| else: | |
| # 如果硬件处理失败,回退软编 | |
| try: | |
| hw_decode_args, vf_string, vcodec, enc_args = self._get_hard_cfg(subtitles_file) | |
| cmd2[cmd2.index('-c:v') + 1] = vcodec | |
| self._subprocess(cmd0 + hw_decode_args + cmd1 + [vf_string] + cmd2 + enc_args + cmd3) | |
| except Exception as e: | |
| cmd2[cmd2.index('-c:v') + 1] = f'libx{self.video_codec_num}' | |
| logger.exception(f'硬件处理视频合成失败,回退软编 {e}', exc_info=True) | |
| runffmpeg(cmd0 + cmd1 + subtitle_filter + cmd2 + enc_qua + cmd3, | |
| cmd_dir=self.cfg.cache_folder, force_cpu=True) | |
| except Exception as e: | |
| raise VideoTransError(tr('Error in embedding the final step of the subtitle dubbing')+str(e)) from e | |
| # 复制到目标文件夹 | |
| if Path(tmp_target_mp4).exists(): | |
| try: | |
| self.cfg.targetdir_mp4=self.cfg.targetdir_mp4[:-4]+_video_output_ext | |
| shutil.copy2(tmp_target_mp4, self.cfg.targetdir_mp4) | |
| except Exception: | |
| # 如果移动失败,则尝试直接复制为 0.mp4 or 0.mkv | |
| try: | |
| shutil.copy2(tmp_target_mp4, f'{self.cfg.target_dir}/0{_video_output_ext}') | |
| except Exception as e: | |
| logger.exception(f'再次复制到目标文件夹内 0{_video_output_ext}也失败 {e}', exc_info=True) | |
| raise VideoTransError(tr('Translation successful but transfer failed.', tmp_target_mp4)) from e | |
| # 有可能输出原始音频到目标文件夹的程序仍在执行,但不影响 | |
| while output_source_output is not True: | |
| if app_cfg.exit_soft:return | |
| time.sleep(1) | |
| return | |
| def _get_origin_audio(self, output,duration_ms=0): | |
| # 无需配音的场景下取出原始音频 | |
| if self.video_info.get('streams_audio', 0) == 0: | |
| # 无音频流 | |
| return | |
| cmd = [ | |
| "-y", | |
| "-i", | |
| self.cfg.name, | |
| "-vn" | |
| ] | |
| # 如果视频时长大于音频 100ms,需延长 | |
| v_a_offset=int(self.video_info['time'])-duration_ms | |
| if duration_ms>0 and v_a_offset>100: | |
| cmd.extend(['-af', f'apad=pad_dur={v_a_offset/1000.0}']) | |
| cmd.extend(['-c:a', 'aac', '-b:a', '128k',output]) | |
| return runffmpeg(cmd) | |
| # ffmpeg进度日志 | |
| def _hebing_pro(self, protxt) -> None: | |
| while 1: | |
| if app_cfg.exit_soft or self.hasend or self.uuid in app_cfg.stoped_uuid_set: return | |
| content = read_last_n_lines(protxt) | |
| if not content: | |
| time.sleep(0.5) | |
| continue | |
| if content[-1] == 'progress=end': | |
| return | |
| idx = len(content) - 1 | |
| end_time = "00:00:00" | |
| while idx > 0: | |
| if content[idx].startswith('out_time='): | |
| end_time = content[idx].split('=')[1].strip() | |
| break | |
| idx -= 1 | |
| self.signal(text=tr('kaishihebing') + f' {end_time}') | |
| time.sleep(0.5) | |
| # 视频合成时,返回可用的硬件解码参数、字幕嵌入参数、视频编码参数、质量相关参数 | |
| def _get_hard_cfg(self, subtitles_file=None, codec=None): | |
| os_name = platform.system() | |
| if not app_cfg.video_codec: | |
| app_cfg.video_codec = get_video_codec() | |
| # 仅用于确定编码器部分,具体 264或265由 codec 决定 | |
| hw_type = app_cfg.video_codec | |
| logger.debug(f'原始{hw_type=}') | |
| if '_' in hw_type: | |
| _hw_type_list = hw_type.lower().split('_') | |
| if _hw_type_list[0] == 'vaapi': | |
| hw_type = 'vaapi' | |
| else: | |
| hw_type = _hw_type_list[1] | |
| logger.debug(f'整理后{hw_type=}') | |
| # 硬字幕由于是软过滤,必须先在内存中压制。 | |
| # 不同的硬件编码器可能需要在软过滤后,将画面重新上传到显存(hwupload) | |
| codec = f'{self.video_codec_num}' if not codec else codec | |
| vcodec = f"libx{codec}" | |
| _crf = f'{settings.get("crf", 23)}' | |
| # 全局参数,硬件解码相关 | |
| global_args = [] | |
| # 硬字幕嵌入参数,软字幕忽略 | |
| vf_string = f"[0:v]subtitles=filename='{subtitles_file}'[v_out]" | |
| # 硬件兼容有限,防止出错 | |
| _preset = settings.get('preset', 'medium') | |
| if 'fast' in _preset: | |
| _preset = 'fast' | |
| elif 'slow' in _preset: | |
| _preset = 'slow' | |
| if _preset not in ['fast', 'slow', 'medium']: | |
| _preset = 'medium' | |
| enc_args = ['-crf', _crf, '-preset', _preset] | |
| # --- 参数映射表 --- | |
| PRESET_MAP = { | |
| # NVENC: p1 (最快) - p7 (最慢/质量最好) | |
| 'nvenc': {'fast': 'p2', 'medium': 'p4', 'slow': 'p7'}, | |
| # QSV: veryfast, faster, fast, medium, slow, slower, veryslow | |
| 'qsv': {'fast': 'fast', 'medium': 'medium', 'slow': 'slow'}, | |
| # AMF: speed, balanced, quality | |
| 'amf': {'fast': 'speed', 'medium': 'balanced', 'slow': 'quality'}, | |
| # VAAPI: 通常也接受 standard presets | |
| 'vaapi': {'fast': 'fast', 'medium': 'medium', 'slow': 'slow'}, | |
| # VideoToolbox: 通常不支持 -preset 参数,留空以跳过处理 | |
| 'videotoolbox': None | |
| } | |
| # --- Nvidia (NVENC) --- | |
| if hw_type in ['nvenc']: | |
| vcodec = "h264_nvenc" if codec == '264' else "hevc_nvenc" | |
| # nvenc 使用 -cq (Constant Quality) 替代 crf,p4 预设在速度和质量间平衡较好 | |
| enc_args = ['-cq', _crf, '-preset', PRESET_MAP.get('nvenc').get(_preset, 'p4')] | |
| # 优先硬件解码 | |
| if settings.get('hw_decode'): | |
| global_args = ['-hwaccel', 'cuda', '-hwaccel_output_format', 'cuda'] | |
| vf_string = f"[0:v]hwdownload,format=nv12,subtitles=filename='{subtitles_file}',hwupload_cuda[v_out]" | |
| else: | |
| vf_string = f"[0:v]subtitles=filename='{subtitles_file}'[v_out]" | |
| return global_args, vf_string, vcodec, enc_args | |
| # --- Mac (VideoToolbox) --- | |
| if hw_type in ['videotoolbox']: | |
| vcodec = "h264_videotoolbox" if codec == '264' else "hevc_videotoolbox" | |
| # videotoolbox 质量控制,通常用 -q:v (范围约在 40-60 之间视觉无损) | |
| quality = int(100 - (int(_crf) * 1.4)) | |
| enc_args = ['-q:v', f'{int(max(1, min(quality, 100)))}'] | |
| return global_args, vf_string, vcodec, enc_args | |
| # --- Intel (QSV) & AMD (AMF) --- | |
| if hw_type in ['qsv', 'amf', 'vaapi']: | |
| if os_name == 'Linux': | |
| # 【Linux 特殊处理】 | |
| # 在 Linux 下,Intel 和 AMD 开源驱动通常统一走 VAAPI 接口 | |
| devices = glob.glob('/dev/dri/renderD*') | |
| device = devices[0] if devices else '/dev/dri/renderD128' | |
| if settings.get('hw_decode'): | |
| global_args = ['-hwaccel', 'vaapi', '-hwaccel_device', device, '-hwaccel_output_format', 'vaapi'] | |
| vf_string = f"[0:v]hwdownload,format=nv12,subtitles=filename='{subtitles_file}',format=nv12,hwupload[v_out]" | |
| else: | |
| global_args = [ | |
| '-init_hw_device', f'vaapi=vaapi:{device}' | |
| ] | |
| vf_string = f"[0:v]subtitles=filename='{subtitles_file}',format=nv12,hwupload[v_out]" | |
| vcodec = "h264_vaapi" if codec == '264' else "hevc_vaapi" | |
| enc_args = ['-qp', _crf, '-preset', PRESET_MAP.get('vaapi').get(_preset, 'medium')] | |
| return global_args, vf_string, vcodec, enc_args | |
| # VAAPI 要求在软滤镜(字幕)处理完后,转换像素格式并上传到显存 | |
| # Windows 环境 | |
| if hw_type in ['qsv']: | |
| vcodec = "h264_qsv" if codec == '264' else "hevc_qsv" | |
| # QSV 使用 ICQ 模式 (Intelligent Constant Quality) | |
| enc_args = ['-global_quality', _crf, '-preset', PRESET_MAP.get('qsv').get(_preset, 'medium')] | |
| else: | |
| vcodec = "h264_amf" if codec == '264' else "hevc_amf" | |
| # AMF 使用恒定质量参数 (CQP) | |
| enc_args = ['-rc', 'cqp', '-qp_p', _crf, '-qp_i', _crf, '-quality', | |
| PRESET_MAP.get('amf').get(_preset, 'balanced')] | |
| return global_args, vf_string, vcodec, enc_args | |
| return global_args, vf_string, vcodec, enc_args | |
| def _subprocess(self, cmd): | |
| logger.debug(f'[尝试硬件编解码执行命令]\n{" ".join(cmd)}\n') | |
| try: | |
| if app_cfg.exit_soft: return | |
| cmd = ["ffmpeg", '-nostdin'] + cmd | |
| subprocess.run( | |
| cmd, | |
| # stdout=subprocess.PIPE, | |
| # stderr=subprocess.PIPE, | |
| encoding="utf-8", | |
| errors='ignore', | |
| check=True, | |
| text=True, | |
| capture_output=True, | |
| creationflags=subprocess.CREATE_NO_WINDOW if sys.platform == 'win32' else 0, | |
| cwd=self.cfg.cache_folder | |
| ) | |
| return True | |
| except subprocess.CalledProcessError as e: | |
| raise FFmpegError(f"尝试使用硬件执行命令出错[CalledProcessError]:{e.stderr}\n{e.stdout},{e}") from e | |