diff --git a/runtime-data/references/.gitkeep b/runtime-data/references/.gitkeep new file mode 100644 index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc --- /dev/null +++ b/runtime-data/references/.gitkeep @@ -0,0 +1 @@ + diff --git a/runtime/experimental/__init__.py b/runtime/experimental/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..acd50f841c65270a000eb50ba313b8a5a1f593b3 --- /dev/null +++ b/runtime/experimental/__init__.py @@ -0,0 +1 @@ +"""Experimental code kept separate from the pinned Fish Speech baseline.""" diff --git a/runtime/experimental/codec.py b/runtime/experimental/codec.py new file mode 100644 index 0000000000000000000000000000000000000000..352107b2a023cee09fa40eed4c4cf024238dc750 --- /dev/null +++ b/runtime/experimental/codec.py @@ -0,0 +1,336 @@ +"""Project-local inference memory optimizations for the S2-Pro DAC codec. + +The pinned Fish Speech source remains unchanged. This module builds the same +codec and loads the same checkpoint, then removes buffers that are unnecessary +for the window-limited inference path before the model is moved to CUDA. +""" + +from __future__ import annotations + +import gc +import io +import math +import os +import threading +import time +from pathlib import Path +from typing import Any + +import torch + +os.environ.setdefault("TEMPORARILY_DISABLE_PROTOBUF_VERSION_CHECK", "true") + +from fish_speech.models.dac.modded_dac import DAC + + +def load_reference_audio_soundfile( + reference_audio: bytes | str | Path, + sample_rate: int, +): + """Decode API reference audio without TorchCodec. + + Torch 2.11 routes ``torchaudio.load`` through optional TorchCodec. The + pinned container already includes SoundFile, which supports the WAV/FLAC + inputs accepted by this service, so no environment package mutation is + needed. + """ + + import numpy as np + import soundfile as sf + import torchaudio + + source = ( + io.BytesIO(reference_audio) + if isinstance(reference_audio, bytes) + else reference_audio + ) + audio, original_rate = sf.read(source, dtype="float32", always_2d=True) + mono = np.asarray(audio.mean(axis=1), dtype=np.float32) + if original_rate != sample_rate: + mono = ( + torchaudio.functional.resample( + torch.from_numpy(mono), + original_rate, + sample_rate, + ) + .contiguous() + .numpy() + ) + return mono + + +@torch.inference_mode() +def warm_reference_encoder( + codec: torch.nn.Module, + device: str | torch.device, + seconds: float = 1.0, +) -> dict[str, Any]: + """Prime lazy codec state before the first user reference is cached. + + The staged BF16 encoder produces a different discrete code sequence on its + first CUDA pass. A discarded silence pass makes subsequent encodes bit + stable, preventing the first uploaded voice from being conditioned on an + avoidable cold-start code path. + """ + + if seconds <= 0: + raise ValueError("Reference warmup duration must be positive") + target = torch.device(device) + sample_rate = int(codec.sample_rate) + samples = int(round(sample_rate * seconds)) + audio = torch.zeros((1, samples), dtype=torch.float32) + lengths = torch.tensor([samples], device=target, dtype=torch.long) + started = time.perf_counter() + codes = codec.encode(audio, lengths)[0][0].cpu() + return { + "input": "digital_silence", + "seconds": seconds, + "samples": samples, + "code_frames": int(codes.shape[-1]), + "elapsed_seconds": time.perf_counter() - started, + } + + +class StagedReferenceCodec(DAC): + """DAC with decode modules resident and reference-only modules staged.""" + + @property + def device(self) -> torch.device: + return self._decode_device + + def configure_reference_staging( + self, + decode_device: str | torch.device, + offload_device: str | torch.device = "cpu", + ) -> None: + self._decode_device = torch.device(decode_device) + self._reference_offload_device = torch.device(offload_device) + self._reference_lock = threading.Lock() + + # These modules are used by ``from_indices`` and remain resident. + self.quantizer.semantic_quantizer.to(self._decode_device) + self.quantizer.quantizer.to(self._decode_device) + self.quantizer.post_module.to(self._decode_device) + self.quantizer.upsample.to(self._decode_device) + self.decoder.to(self._decode_device) + + # These modules are required only while a new reference is encoded. + self.encoder.to(self._reference_offload_device) + self.quantizer.downsample.to(self._reference_offload_device) + self.quantizer.pre_module.to(self._reference_offload_device) + + @torch.inference_mode() + def encode( + self, + audio_data: torch.Tensor, + audio_lengths: torch.Tensor | None = None, + n_quantizers: int | None = None, + **kwargs, + ): + """Encode reference codes, staging only the required modules on CUDA.""" + + if not hasattr(self, "_reference_lock"): + return super().encode( + audio_data, + audio_lengths=audio_lengths, + n_quantizers=n_quantizers, + **kwargs, + ) + + with self._reference_lock: + reference_modules = ( + self.encoder, + self.quantizer.downsample, + self.quantizer.pre_module, + ) + for module in reference_modules: + module.to(self._decode_device) + + try: + dtype = next(self.encoder.parameters()).dtype + audio_data = audio_data.to(device=self._decode_device, dtype=dtype) + if audio_data.ndim == 2: + audio_data = audio_data.unsqueeze(1) + length = audio_data.shape[-1] + right_pad = ( + math.ceil(length / self.frame_length) * self.frame_length - length + ) + audio_data = torch.nn.functional.pad(audio_data, (0, right_pad)) + if audio_lengths is None: + audio_lengths = torch.tensor( + [length + right_pad], + device=self._decode_device, + dtype=torch.long, + ) + else: + audio_lengths = audio_lengths.to(self._decode_device) + + z = self.encoder(audio_data) + z = self.quantizer.downsample(z) + z = self.quantizer.pre_module(z) + semantic_z, semantic_codes, *_ = self.quantizer.semantic_quantizer(z) + residual_z = z - semantic_z + _, residual_codes, *_ = self.quantizer.quantizer( + residual_z, + n_quantizers=n_quantizers, + ) + indices = torch.cat([semantic_codes, residual_codes], dim=1) + indices_lens = torch.ceil(audio_lengths / self.frame_length).long() + finally: + if self._decode_device.type == "cuda": + torch.cuda.synchronize(self._decode_device) + for module in reference_modules: + module.to(self._reference_offload_device) + if self._decode_device.type == "cuda": + torch.cuda.empty_cache() + + return indices, indices_lens + + +def _tensor_bytes(tensor: torch.Tensor | None) -> int: + if tensor is None: + return 0 + return tensor.numel() * tensor.element_size() + + +@torch.inference_mode() +def compact_codec_inference_buffers(codec: torch.nn.Module) -> dict[str, Any]: + """Remove dead causal masks and bound RoPE tables to configured limits. + + ``WindowLimitedTransformer.forward`` always constructs an exact mask for + the current input and passes it to its parent implementation. Therefore the + inherited 32768-square causal mask is not read on this path. Its RoPE table + is used, but the configured block size is the model's supported inference + limit and is far smaller than the inherited 327680-frame table. + """ + + from fish_speech.models.dac.modded_dac import WindowLimitedTransformer + + records: list[dict[str, Any]] = [] + saved_bytes = 0 + for name, module in codec.named_modules(): + if not isinstance(module, WindowLimitedTransformer): + continue + + causal_mask = module.causal_mask + freqs_cis = module.freqs_cis + if freqs_cis is None: + raise RuntimeError(f"Codec transformer {name} has no RoPE table") + + frame_limit = int(module.config.block_size) + if frame_limit <= 0 or frame_limit > freqs_cis.shape[0]: + raise RuntimeError( + f"Invalid codec RoPE limit for {name}: {frame_limit} " + f"of {freqs_cis.shape[0]}" + ) + + before_mask_bytes = _tensor_bytes(causal_mask) + before_rope_bytes = _tensor_bytes(freqs_cis) + device = freqs_cis.device + module.causal_mask = torch.empty(0, dtype=torch.bool, device=device) + module.freqs_cis = freqs_cis[:frame_limit].clone() + after_rope_bytes = _tensor_bytes(module.freqs_cis) + module._compact_inference_frame_limit = frame_limit + + records.append( + { + "module": name, + "frame_limit": frame_limit, + "removed_causal_mask_bytes": before_mask_bytes, + "rope_bytes_before": before_rope_bytes, + "rope_bytes_after": after_rope_bytes, + } + ) + saved_bytes += before_mask_bytes + before_rope_bytes - after_rope_bytes + + if len(records) != 3: + raise RuntimeError( + f"Expected three window-limited codec transformers, found {len(records)}" + ) + + report = { + "policy": "compact_windowed_inference_buffers", + "windowed_transformers": len(records), + "theoretical_saved_bytes": saved_bytes, + "records": records, + } + codec._compact_inference_buffers_report = report + return report + + +@torch.inference_mode() +def load_compact_codec_model( + config_name: str, + checkpoint_path: str | Path, + device: str | torch.device = "cuda:0", + precision: torch.dtype = torch.bfloat16, + offload_reference: bool = False, +) -> torch.nn.Module: + """Load the pinned codec with compact buffers before CUDA placement.""" + + from hydra.utils import instantiate + from omegaconf import OmegaConf + + from fish_speech.models.dac import modded_dac as modded_dac_module + + config_path = ( + Path(modded_dac_module.__file__).resolve().parents[2] + / "configs" + / f"{config_name}.yaml" + ) + cfg = OmegaConf.load(config_path) + if offload_reference: + cfg._target_ = "experimental.codec.StagedReferenceCodec" + + codec = instantiate(cfg) + state_dict = torch.load( + checkpoint_path, + map_location="cpu", + mmap=True, + weights_only=True, + ) + if "state_dict" in state_dict: + state_dict = state_dict["state_dict"] + if any("generator" in key for key in state_dict): + state_dict = { + key.replace("generator.", ""): value + for key, value in state_dict.items() + if "generator." in key + } + + load_result = codec.load_state_dict(state_dict, strict=False, assign=True) + unexpected = [ + key + for key in load_result.unexpected_keys + if not key.endswith(("causal_mask", "freqs_cis")) + ] + if load_result.missing_keys or unexpected: + raise RuntimeError( + "Unexpected compact codec checkpoint mismatch: " + f"missing={load_result.missing_keys[:5]}, unexpected={unexpected[:5]}" + ) + + report = compact_codec_inference_buffers(codec) + codec.eval() + codec.to(dtype=precision) + if offload_reference: + codec.configure_reference_staging(device) + report["reference_path"] = "staged_from_cpu_to_decode_device" + else: + codec.to(device=device) + report["reference_path"] = "resident_on_decode_device" + codec._compact_inference_buffers_report = report + del state_dict + gc.collect() + if torch.cuda.is_available() and torch.device(device).type == "cuda": + torch.cuda.empty_cache() + return codec + + +__all__ = [ + "StagedReferenceCodec", + "compact_codec_inference_buffers", + "load_compact_codec_model", + "load_reference_audio_soundfile", + "warm_reference_encoder", +] diff --git a/runtime/native/LICENSE b/runtime/native/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..8847637261d5706d1dc56b38b252b0abbac54218 --- /dev/null +++ b/runtime/native/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Support. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright (c) 2025 Comfy Org. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/runtime/native/direct_w4a4_m1/direct_w4a4_m1.cpp b/runtime/native/direct_w4a4_m1/direct_w4a4_m1.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e9913b77f95d270e746cc63bb074fd604ecbf0c2 --- /dev/null +++ b/runtime/native/direct_w4a4_m1/direct_w4a4_m1.cpp @@ -0,0 +1,104 @@ +#include "direct_w4a4_m1.h" + +#include + +#include + +namespace { + +torch::Tensor direct_w4a4_m1_linear( + const torch::Tensor& activation_qdata, + const torch::Tensor& activation_block_scales, + const torch::Tensor& activation_tensor_scale, + const torch::Tensor& weight_qdata, + const torch::Tensor& weight_block_scales, + const torch::Tensor& weight_tensor_scale, + const std::optional& bias) { + TORCH_CHECK( + activation_qdata.is_cuda() && + activation_qdata.scalar_type() == at::kByte && + activation_qdata.dim() == 2 && activation_qdata.is_contiguous(), + "activation qdata must be contiguous CUDA uint8 [padded_M,K/2]"); + TORCH_CHECK( + activation_qdata.size(0) >= 1, + "direct W4A4 M=1 requires at least one packed activation row"); + TORCH_CHECK( + activation_block_scales.is_cuda() && + activation_block_scales.dim() == 2 && + activation_block_scales.is_contiguous() && + activation_block_scales.element_size() == 1, + "activation block scales must be contiguous CUDA byte-sized [padded_M,padded_K/16]"); + TORCH_CHECK( + activation_tensor_scale.is_cuda() && + activation_tensor_scale.scalar_type() == at::kFloat && + activation_tensor_scale.numel() == 1 && + activation_tensor_scale.is_contiguous(), + "activation tensor scale must be one contiguous CUDA float32 value"); + TORCH_CHECK( + weight_qdata.is_cuda() && weight_qdata.scalar_type() == at::kByte && + weight_qdata.dim() == 2 && weight_qdata.is_contiguous(), + "weight qdata must be contiguous CUDA uint8 [N,K/2]"); + TORCH_CHECK( + weight_block_scales.is_cuda() && weight_block_scales.dim() == 2 && + weight_block_scales.is_contiguous() && + weight_block_scales.element_size() == 1, + "weight block scales must be contiguous CUDA byte-sized [padded_N,padded_K/16]"); + TORCH_CHECK( + weight_tensor_scale.is_cuda() && + weight_tensor_scale.scalar_type() == at::kFloat && + weight_tensor_scale.numel() == 1 && + weight_tensor_scale.is_contiguous(), + "weight tensor scale must be one contiguous CUDA float32 value"); + TORCH_CHECK( + activation_qdata.device() == activation_block_scales.device() && + activation_qdata.device() == activation_tensor_scale.device() && + activation_qdata.device() == weight_qdata.device() && + activation_qdata.device() == weight_block_scales.device() && + activation_qdata.device() == weight_tensor_scale.device(), + "all direct W4A4 tensors must use the same CUDA device"); + + const int64_t in_features = activation_qdata.size(1) * 2; + const int64_t out_features = weight_qdata.size(0); + TORCH_CHECK( + in_features > 0 && in_features % 32 == 0, + "direct W4A4 M=1 requires K divisible by 32"); + TORCH_CHECK( + weight_qdata.size(1) == activation_qdata.size(1), + "activation and weight packed K dimensions differ"); + TORCH_CHECK(out_features > 0, "direct W4A4 M=1 requires positive N"); + TORCH_CHECK( + activation_block_scales.size(0) >= 1 && + activation_block_scales.size(1) >= in_features / 16, + "activation block-scale tensor is too small"); + TORCH_CHECK( + weight_block_scales.size(0) >= out_features && + weight_block_scales.size(1) >= in_features / 16, + "weight block-scale tensor is too small"); + + if (bias.has_value()) { + const auto& value = *bias; + TORCH_CHECK( + value.is_cuda() && value.scalar_type() == at::kBFloat16 && + value.dim() == 1 && value.is_contiguous() && + value.numel() == out_features && + value.device() == activation_qdata.device(), + "bias must be contiguous CUDA bfloat16 [N]"); + } + return direct_w4a4_m1_linear_cuda( + activation_qdata, + activation_block_scales, + activation_tensor_scale, + weight_qdata, + weight_block_scales, + weight_tensor_scale, + bias); +} + +} // namespace + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, module) { + module.def( + "linear", + &direct_w4a4_m1_linear, + "Direct packed-NVFP4 activation x packed-NVFP4 weight M=1 linear"); +} diff --git a/runtime/native/rmsnorm_nvfp4_m1/rmsnorm_nvfp4_m1.cpp b/runtime/native/rmsnorm_nvfp4_m1/rmsnorm_nvfp4_m1.cpp new file mode 100644 index 0000000000000000000000000000000000000000..6e77ff5a6cccef8a6d7c99c5dc3781121a21ed2e --- /dev/null +++ b/runtime/native/rmsnorm_nvfp4_m1/rmsnorm_nvfp4_m1.cpp @@ -0,0 +1,38 @@ +#include "rmsnorm_nvfp4_m1.h" + +#include + +#include + +namespace { + +std::vector rmsnorm_nvfp4_m1( + const torch::Tensor& input, + const torch::Tensor& weight, + double epsilon) { + TORCH_CHECK( + input.is_cuda() && input.scalar_type() == at::kBFloat16 && + input.dim() == 2 && input.size(0) == 1 && input.is_contiguous(), + "RMSNorm input must be contiguous CUDA bfloat16 [1,K]"); + TORCH_CHECK( + weight.is_cuda() && weight.scalar_type() == at::kBFloat16 && + weight.dim() == 1 && weight.is_contiguous(), + "RMSNorm weight must be contiguous CUDA bfloat16 [K]"); + TORCH_CHECK( + weight.device() == input.device() && weight.numel() == input.size(1), + "RMSNorm input and weight dimensions/devices differ"); + TORCH_CHECK( + input.size(1) > 0 && input.size(1) % 32 == 0, + "fused RMSNorm-to-NVFP4 requires K divisible by 32"); + TORCH_CHECK(epsilon > 0.0, "RMSNorm epsilon must be positive"); + return rmsnorm_nvfp4_m1_cuda(input, weight, epsilon); +} + +} // namespace + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, module) { + module.def( + "quantize", + &rmsnorm_nvfp4_m1, + "Fused Qwen RMSNorm to packed NVFP4 at logical M=1"); +} diff --git a/runtime/native/rmsnorm_nvfp4_m1/rmsnorm_nvfp4_m1.cu b/runtime/native/rmsnorm_nvfp4_m1/rmsnorm_nvfp4_m1.cu new file mode 100644 index 0000000000000000000000000000000000000000..b054e7d035546ccddf4598296ca15701cc966b24 --- /dev/null +++ b/runtime/native/rmsnorm_nvfp4_m1/rmsnorm_nvfp4_m1.cu @@ -0,0 +1,160 @@ +#include "rmsnorm_nvfp4_m1.h" + +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace { + +constexpr int kThreads = 256; +constexpr int kFp4BlockElements = 16; +constexpr int kScaleTileOuter = 128; +constexpr float kFp4Max = 6.0f; +constexpr float kFp8Max = 448.0f; +constexpr float kCombinedMax = kFp4Max * kFp8Max; + +__device__ __forceinline__ size_t scale_offset( + int outer, + int inner_scale, + int scale_inner_dim) { + const int outer_tile = outer / kScaleTileOuter; + const int local_outer = outer % kScaleTileOuter; + const int local_inner = inner_scale & 3; + const int inner_tile_start = inner_scale - local_inner; + const size_t tile_base = + static_cast(inner_tile_start + outer_tile * scale_inner_dim) * + kScaleTileOuter; + return tile_base + static_cast(local_outer & 31) * 16 + + static_cast(local_outer >> 5) * 4 + local_inner; +} + +__device__ __forceinline__ float e4m3_to_float(uint8_t raw) { + const __half_raw half_raw = __nv_cvt_fp8_to_halfraw(raw, __NV_E4M3); + return __half2float(static_cast<__half>(half_raw)); +} + +__device__ __forceinline__ float qwen_rmsnorm_value( + const __nv_bfloat16* input, + const __nv_bfloat16* weight, + int index, + float inverse_rms) { + const float normalized = __bfloat162float(input[index]) * inverse_rms; + const __nv_bfloat16 normalized_bf16 = __float2bfloat16_rn(normalized); + return __bfloat162float(__float2bfloat16_rn( + __bfloat162float(normalized_bf16) * __bfloat162float(weight[index]))); +} + +__global__ void rmsnorm_nvfp4_m1_kernel( + const __nv_bfloat16* __restrict__ input, + const __nv_bfloat16* __restrict__ weight, + uint8_t* __restrict__ qdata, + uint8_t* __restrict__ block_scales, + float* __restrict__ tensor_scale, + int k, + int scale_inner_dim, + float epsilon) { + __shared__ float reduction[kThreads]; + const int tid = threadIdx.x; + float sum_square = 0.0f; + for (int index = tid; index < k; index += kThreads) { + const float value = __bfloat162float(input[index]); + sum_square = fmaf(value, value, sum_square); + } + reduction[tid] = sum_square; + __syncthreads(); + for (int offset = kThreads / 2; offset > 0; offset >>= 1) { + if (tid < offset) { + reduction[tid] += reduction[tid + offset]; + } + __syncthreads(); + } + const float inverse_rms = rsqrtf(reduction[0] / static_cast(k) + epsilon); + + float local_max = 0.0f; + for (int index = tid; index < k; index += kThreads) { + local_max = fmaxf( + local_max, + fabsf(qwen_rmsnorm_value(input, weight, index, inverse_rms))); + } + reduction[tid] = local_max; + __syncthreads(); + for (int offset = kThreads / 2; offset > 0; offset >>= 1) { + if (tid < offset) { + reduction[tid] = fmaxf(reduction[tid], reduction[tid + offset]); + } + __syncthreads(); + } + // TensorCoreNVFP4Layout computes the default scale from a BF16 amax, so + // the division result is rounded to BF16 before Params converts it to F32. + const float global_scale = __bfloat162float(__float2bfloat16_rn( + __fdiv_rn(reduction[0], kCombinedMax))); + if (tid == 0) { + tensor_scale[0] = global_scale; + } + __syncthreads(); + + const int block_count = k / kFp4BlockElements; + for (int block = tid; block < block_count; block += kThreads) { + const int base = block * kFp4BlockElements; + float values[kFp4BlockElements]; + float block_max = 0.0f; +#pragma unroll + for (int element = 0; element < kFp4BlockElements; ++element) { + values[element] = qwen_rmsnorm_value( + input, weight, base + element, inverse_rms); + block_max = fmaxf(block_max, fabsf(values[element])); + } + float scaled_block_scale = (block_max / kFp4Max) / global_scale; + scaled_block_scale = fminf(scaled_block_scale, kFp8Max); + const uint8_t raw_block_scale = static_cast( + __nv_cvt_float_to_fp8( + scaled_block_scale, __NV_SATFINITE, __NV_E4M3)); + block_scales[scale_offset(0, block, scale_inner_dim)] = raw_block_scale; + const float total_scale = global_scale * e4m3_to_float(raw_block_scale); +#pragma unroll + for (int pair = 0; pair < kFp4BlockElements / 2; ++pair) { + const float even = values[pair * 2] / total_scale; + const float odd = values[pair * 2 + 1] / total_scale; + const float2 arguments = make_float2(odd, even); + qdata[base / 2 + pair] = static_cast( + __nv_cvt_float2_to_fp4x2(arguments, __NV_E2M1, cudaRoundNearest)); + } + } +} + +} // namespace + +std::vector rmsnorm_nvfp4_m1_cuda( + const torch::Tensor& input, + const torch::Tensor& weight, + double epsilon) { + const auto device = input.device(); + c10::cuda::CUDAGuard guard(device); + const int64_t k = input.size(1); + const int64_t scale_cols = (k / 16 + 3) / 4 * 4; + torch::Tensor qdata = torch::zeros( + {16, k / 2}, input.options().dtype(at::kByte)); + torch::Tensor block_scales = torch::zeros( + {128, scale_cols}, + input.options().dtype(at::ScalarType::Float8_e4m3fn)); + torch::Tensor tensor_scale = torch::empty( + {}, input.options().dtype(at::kFloat)); + const auto stream = at::cuda::getCurrentCUDAStream(device.index()).stream(); + rmsnorm_nvfp4_m1_kernel<<<1, kThreads, 0, stream>>>( + reinterpret_cast(input.data_ptr()), + reinterpret_cast(weight.data_ptr()), + reinterpret_cast(qdata.data_ptr()), + reinterpret_cast(block_scales.data_ptr()), + reinterpret_cast(tensor_scale.data_ptr()), + static_cast(k), + static_cast(scale_cols), + static_cast(epsilon)); + C10_CUDA_KERNEL_LAUNCH_CHECK(); + return {qdata, block_scales, tensor_scale}; +} diff --git a/runtime/native/rmsnorm_nvfp4_m1/rmsnorm_nvfp4_m1.h b/runtime/native/rmsnorm_nvfp4_m1/rmsnorm_nvfp4_m1.h new file mode 100644 index 0000000000000000000000000000000000000000..a3fe1732feac8e1104cb3aa79b5536e46191a0bc --- /dev/null +++ b/runtime/native/rmsnorm_nvfp4_m1/rmsnorm_nvfp4_m1.h @@ -0,0 +1,10 @@ +#pragma once + +#include + +#include + +std::vector rmsnorm_nvfp4_m1_cuda( + const torch::Tensor& input, + const torch::Tensor& weight, + double epsilon); diff --git a/runtime/native/smallm_gemv/smallm_gemv.cpp b/runtime/native/smallm_gemv/smallm_gemv.cpp new file mode 100644 index 0000000000000000000000000000000000000000..9173b030595e889469487ab37cfccb1e3464c353 --- /dev/null +++ b/runtime/native/smallm_gemv/smallm_gemv.cpp @@ -0,0 +1,86 @@ +#include "smallm_gemv.h" + +#include + +#include + +namespace { + +torch::Tensor smallm_nvfp4_linear( + const torch::Tensor& input, + const torch::Tensor& packed_weight, + const torch::Tensor& weight_block_scales, + const torch::Tensor& weight_tensor_scale, + const std::optional& bias) { + TORCH_CHECK(input.is_cuda(), "small-M GEMV requires a CUDA input"); + TORCH_CHECK( + input.scalar_type() == at::kBFloat16, + "small-M GEMV input must be bfloat16"); + TORCH_CHECK( + input.dim() >= 1 && input.is_contiguous(), + "small-M GEMV input must be contiguous"); + TORCH_CHECK( + packed_weight.is_cuda() && packed_weight.scalar_type() == at::kByte && + packed_weight.dim() == 2 && packed_weight.is_contiguous(), + "packed weight must be contiguous CUDA uint8 [N,K/2]"); + TORCH_CHECK( + weight_block_scales.is_cuda() && weight_block_scales.dim() == 2 && + weight_block_scales.is_contiguous() && + weight_block_scales.element_size() == 1, + "weight block scales must be contiguous CUDA byte-sized [padded_N,padded_K/16]"); + TORCH_CHECK( + weight_tensor_scale.is_cuda() && + weight_tensor_scale.scalar_type() == at::kFloat && + weight_tensor_scale.numel() == 1 && + weight_tensor_scale.is_contiguous(), + "weight tensor scale must be one contiguous CUDA float32 value"); + TORCH_CHECK( + input.device() == packed_weight.device() && + input.device() == weight_block_scales.device() && + input.device() == weight_tensor_scale.device(), + "all small-M GEMV tensors must use the same CUDA device"); + + const int64_t out_features = packed_weight.size(0); + const int64_t in_features = packed_weight.size(1) * 2; + TORCH_CHECK( + input.size(-1) == in_features, + "small-M GEMV expected input width ", + in_features, + " but got ", + input.size(-1)); + TORCH_CHECK( + in_features > 0 && in_features % 32 == 0, + "small-M GEMV requires K divisible by 32"); + TORCH_CHECK( + out_features > 0, + "small-M GEMV requires positive N"); + TORCH_CHECK( + weight_block_scales.size(0) >= out_features && + weight_block_scales.size(1) >= in_features / 16, + "weight block scale tensor is too small"); + + if (bias.has_value()) { + const auto& value = *bias; + TORCH_CHECK( + value.is_cuda() && value.scalar_type() == at::kBFloat16 && + value.dim() == 1 && value.is_contiguous() && + value.numel() == out_features && + value.device() == input.device(), + "bias must be contiguous CUDA bfloat16 [N]"); + } + return smallm_nvfp4_linear_cuda( + input, + packed_weight, + weight_block_scales, + weight_tensor_scale, + bias); +} + +} // namespace + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, module) { + module.def( + "linear", + &smallm_nvfp4_linear, + "Fused BF16-activation x packed-NVFP4-weight small-M GEMV"); +} diff --git a/runtime/native/smallm_gemv/smallm_gemv.cu b/runtime/native/smallm_gemv/smallm_gemv.cu new file mode 100644 index 0000000000000000000000000000000000000000..d49cd7d789f872c0971b2221306bd5478a35c659 --- /dev/null +++ b/runtime/native/smallm_gemv/smallm_gemv.cu @@ -0,0 +1,171 @@ +#include "smallm_gemv.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace { + +constexpr int kWarpSize = 32; +constexpr int kWarpsPerBlock = 8; +constexpr int kThreads = kWarpSize * kWarpsPerBlock; +constexpr int kFp4BlockElements = 16; +constexpr int kScaleTileOuter = 128; + +__device__ __forceinline__ size_t scale_offset( + int outer, + int inner_scale, + int scale_inner_dim) { + const int outer_tile = outer / kScaleTileOuter; + const int local_outer = outer % kScaleTileOuter; + const int local_inner = inner_scale & 3; + const int inner_tile_start = inner_scale - local_inner; + const size_t tile_base = + static_cast( + inner_tile_start + outer_tile * scale_inner_dim) * + kScaleTileOuter; + return tile_base + static_cast(local_outer & 31) * 16 + + static_cast(local_outer >> 5) * 4 + local_inner; +} + +__device__ __forceinline__ float e4m3_to_float(uint8_t raw) { + const __half_raw half_raw = __nv_cvt_fp8_to_halfraw(raw, __NV_E4M3); + return __half2float(static_cast<__half>(half_raw)); +} + +__device__ __forceinline__ float2 e2m1x2_to_float2(uint8_t packed) { + const __half2_raw raw = + __nv_cvt_fp4x2_to_halfraw2(packed, __NV_E2M1); + const __half2 converted(raw); + return __half22float2(converted); +} + +__global__ void smallm_nvfp4_gemv_kernel( + const __nv_bfloat16* __restrict__ input, + const uint8_t* __restrict__ packed_weight, + const uint8_t* __restrict__ weight_block_scales, + const float* __restrict__ weight_tensor_scale, + const __nv_bfloat16* __restrict__ bias, + __nv_bfloat16* __restrict__ output, + int m, + int n, + int k, + int scale_inner_dim) { + const int lane = threadIdx.x & (kWarpSize - 1); + const int warp_in_block = threadIdx.x / kWarpSize; + const int64_t output_linear = + static_cast(blockIdx.x) * kWarpsPerBlock + warp_in_block; + const int64_t output_count = static_cast(m) * n; + if (output_linear >= output_count) { + return; + } + + const int row_m = static_cast(output_linear / n); + const int row_n = static_cast( + output_linear - static_cast(row_m) * n); + const int packed_k = k / 2; + const __nv_bfloat16* input_row = + input + static_cast(row_m) * k; + const uint8_t* weight_row = + packed_weight + static_cast(row_n) * packed_k; + float accumulator = 0.0f; + + for (int pair = lane; pair < packed_k; pair += kWarpSize) { + const uint8_t packed = weight_row[pair]; + const int scale_block = pair / (kFp4BlockElements / 2); + const uint8_t scale_raw = weight_block_scales[ + scale_offset(row_n, scale_block, scale_inner_dim)]; + const float scale = + e4m3_to_float(scale_raw) * weight_tensor_scale[0]; + const float2 weights = e2m1x2_to_float2(packed); + const int input_index = pair * 2; + accumulator = fmaf( + __bfloat162float(input_row[input_index]), + weights.y * scale, + accumulator); + accumulator = fmaf( + __bfloat162float(input_row[input_index + 1]), + weights.x * scale, + accumulator); + } + +#pragma unroll + for (int offset = 16; offset > 0; offset >>= 1) { + accumulator += __shfl_down_sync(0xFFFFFFFF, accumulator, offset); + } + if (lane == 0) { + if (bias != nullptr) { + accumulator += __bfloat162float(bias[row_n]); + } + output[output_linear] = __float2bfloat16_rn(accumulator); + } +} + +} // namespace + +torch::Tensor smallm_nvfp4_linear_cuda( + const torch::Tensor& input, + const torch::Tensor& packed_weight, + const torch::Tensor& weight_block_scales, + const torch::Tensor& weight_tensor_scale, + const std::optional& bias) { + const auto device = input.device(); + c10::cuda::CUDAGuard guard(device); + const int64_t out_features64 = packed_weight.size(0); + const int64_t in_features64 = packed_weight.size(1) * 2; + const int64_t logical_m64 = input.numel() / in_features64; + TORCH_CHECK( + logical_m64 > 0 && + logical_m64 <= static_cast(std::numeric_limits::max()), + "small-M GEMV M is out of range"); + TORCH_CHECK( + out_features64 <= static_cast(std::numeric_limits::max()) && + in_features64 <= static_cast(std::numeric_limits::max()), + "small-M GEMV N or K is out of range"); + const int m = static_cast(logical_m64); + const int n = static_cast(out_features64); + const int k = static_cast(in_features64); + + std::vector output_shape = input.sizes().vec(); + output_shape.back() = out_features64; + torch::Tensor output = torch::empty( + output_shape, + input.options().dtype(at::kBFloat16)); + const int64_t output_count = logical_m64 * out_features64; + const int64_t block_count64 = + (output_count + kWarpsPerBlock - 1) / kWarpsPerBlock; + TORCH_CHECK( + block_count64 <= static_cast(std::numeric_limits::max()), + "small-M GEMV grid is too large"); + + const auto stream = + at::cuda::getCurrentCUDAStream(device.index()).stream(); + const __nv_bfloat16* bias_pointer = + bias.has_value() + ? reinterpret_cast(bias->data_ptr()) + : nullptr; + smallm_nvfp4_gemv_kernel<<< + static_cast(block_count64), + kThreads, + 0, + stream>>>( + reinterpret_cast(input.data_ptr()), + reinterpret_cast(packed_weight.data_ptr()), + reinterpret_cast(weight_block_scales.data_ptr()), + reinterpret_cast(weight_tensor_scale.data_ptr()), + bias_pointer, + reinterpret_cast<__nv_bfloat16*>(output.data_ptr()), + m, + n, + k, + static_cast(weight_block_scales.size(1))); + C10_CUDA_KERNEL_LAUNCH_CHECK(); + return output; +} diff --git a/runtime/native/smallm_gemv/smallm_gemv.h b/runtime/native/smallm_gemv/smallm_gemv.h new file mode 100644 index 0000000000000000000000000000000000000000..13ebd3edcbcdb2644120b86f3e61ddb1da6c70bb --- /dev/null +++ b/runtime/native/smallm_gemv/smallm_gemv.h @@ -0,0 +1,12 @@ +#pragma once + +#include + +#include + +torch::Tensor smallm_nvfp4_linear_cuda( + const torch::Tensor& input, + const torch::Tensor& packed_weight, + const torch::Tensor& weight_block_scales, + const torch::Tensor& weight_tensor_scale, + const std::optional& bias); diff --git a/runtime/web/index.html b/runtime/web/index.html new file mode 100644 index 0000000000000000000000000000000000000000..fe9019f1ce2b9b6951983becc0299e3f39371349 --- /dev/null +++ b/runtime/web/index.html @@ -0,0 +1,234 @@ + + + + + + S2-Pro Quantized TTS + + + +
+
+
V1 · English balanced release
+

Fish Audio S2-Pro NVFP4 Balanced

+

Loading the active checkpoint information…

+
Checking service…
+
+ +
+
+
+
+ + +

The base S2-Pro model can sound flatter than highly expressive TTS systems. Inline instructions may help some prompts, but V1 does not claim to solve that inherited limitation.

+
+
+ + +

Use a clean, consented 10–30 second sample with one speaker and little background noise.

+
+
+ + +

The transcript is required for reliable zero-shot conditioning.

+
+
+ + +
+
+ + +

Use the same seed to A/B sampling. For emotional delivery, put explicit [tag] instructions in the text.

+
+
+ + +
+
+ + +
+
+
+ + Ready. +
+
+
+ + Download WAV +
+
+
Built with Fish Audio. Use only voices you have permission to clone. This derivative is under the Fish Audio Research License; commercial use requires a separate written license from Fish Audio.
+
+ + + + diff --git a/vendor/fish-speech/.github/ISSUE_TEMPLATE/bug_report.yml b/vendor/fish-speech/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000000000000000000000000000000000000..c557f1ffd25d952ae5308d967b5558507416f72b --- /dev/null +++ b/vendor/fish-speech/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,63 @@ +name: "🕷️ Bug report" +description: | + Please follow this template carefully to ensure we can address your issue quickly. + Make sure to provide as much detail as possible, including logs and screenshots. +labels: + - bug +body: + - type: checkboxes + attributes: + label: Self Checks + description: "To ensure timely help, please confirm the following:" + options: + - label: This template is only for bug reports. For questions, please visit [Discussions](https://github.com/fishaudio/fish-speech/discussions). + required: true + - label: I have thoroughly reviewed the project documentation (installation, training, inference) but couldn't find information to solve my problem. [English](https://speech.fish.audio/) [中文](https://speech.fish.audio/zh/) [日本語](https://speech.fish.audio/ja/) [Portuguese (Brazil)](https://speech.fish.audio/pt/) + required: true + - label: I have searched for existing issues, including closed ones. [Search issues](https://github.com/fishaudio/fish-speech/issues) + required: true + - label: I confirm that I am using English to submit this report (我已阅读并同意 [Language Policy](https://github.com/fishaudio/fish-speech/issues/515)). + required: true + - label: "[FOR CHINESE USERS] 请务必使用英文提交 Issue，否则会被关闭。谢谢！:）" + required: true + - label: "Please do not modify this template and fill in all required fields." + required: true + - type: dropdown + attributes: + label: Cloud or Self Hosted + multiple: true + options: + - Cloud + - Self Hosted (Docker) + - Self Hosted (Source) + validations: + required: true + - type: textarea + attributes: + label: Environment Details + description: "Provide details such as OS, Python version, and any relevant software or dependencies." + placeholder: e.g., macOS 13.5, Python 3.10, torch==2.4.1, Gradio 4.44.0 + validations: + required: true + - type: textarea + attributes: + label: Steps to Reproduce + description: | + Include detailed steps, screenshots, and logs. Use the correct markdown syntax for code blocks. + placeholder: | + 1. Run the command `python -m tools.api_client -t "xxxxx"` + 2. Observe the console output error: `ModuleNotFoundError: No module named 'pyaudio'` (with screenshots or logs will be better) + validations: + required: true + - type: textarea + attributes: + label: ✔️ Expected Behavior + placeholder: Describe what you expected to happen. + validations: + required: false + - type: textarea + attributes: + label: ❌ Actual Behavior + placeholder: Describe what actually happened. + validations: + required: false diff --git a/vendor/fish-speech/.github/ISSUE_TEMPLATE/config.yml b/vendor/fish-speech/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000000000000000000000000000000000000..886c238bf64f361460b5eba3906d94de73b092f6 --- /dev/null +++ b/vendor/fish-speech/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: false +contact_links: + - name: "\U0001F4E7 Discussions" + url: https://github.com/fishaudio/fish-speech/discussions + about: General discussions and request help from the community diff --git a/vendor/fish-speech/.github/ISSUE_TEMPLATE/feature_request.yml b/vendor/fish-speech/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000000000000000000000000000000000000..e9fadbb19483f5827352c93aa57e9c20f32d2053 --- /dev/null +++ b/vendor/fish-speech/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,64 @@ +name: "⭐ Feature or enhancement request" +description: Propose something new. +labels: + - enhancement +body: + - type: checkboxes + attributes: + label: Self Checks + description: "To make sure we get to you in time, please check the following :)" + options: + - label: I have thoroughly reviewed the project documentation (installation, training, inference) but couldn't find any relevant information that meets my needs. [English](https://speech.fish.audio/) [中文](https://speech.fish.audio/zh/) [日本語](https://speech.fish.audio/ja/) [Portuguese (Brazil)](https://speech.fish.audio/pt/) + required: true + - label: I have searched for existing issues [search for existing issues]([https://github.com/langgenius/dify/issues](https://github.com/fishaudio/fish-speech/issues)), including closed ones. + required: true + - label: I confirm that I am using English to submit this report (我已阅读并同意 [Language Policy](https://github.com/fishaudio/fish-speech/issues/515)). + required: true + - label: "[FOR CHINESE USERS] 请务必使用英文提交 Issue，否则会被关闭。谢谢！:）" + required: true + - label: "Please do not modify this template :) and fill in all the required fields." + required: true + + - type: textarea + attributes: + label: 1. Is this request related to a challenge you're experiencing? Tell us your story. + description: | + Describe the specific problem or scenario you’re facing in detail. For example: + *"I was trying to use [feature] for [specific task], but encountered [issue]. This was frustrating because...."* + placeholder: Please describe the situation in as much detail as possible. + validations: + required: true + + - type: textarea + attributes: + label: 2. What is your suggested solution? + description: | + Provide a clear description of the feature or enhancement you'd like to propose. + How would this feature solve your issue or improve the project? + placeholder: Describe your idea or proposed solution here. + validations: + required: true + + - type: textarea + attributes: + label: 3. Additional context or comments + description: | + Any other relevant information, links, documents, or screenshots that provide clarity. + Use this section for anything not covered above. + placeholder: Add any extra details here. + validations: + required: false + + - type: checkboxes + attributes: + label: 4. Can you help us with this feature? + description: | + Let us know if you're interested in contributing. This is not a commitment but a way to express interest in collaboration. + options: + - label: I am interested in contributing to this feature. + required: false + + - type: markdown + attributes: + value: | + **Note:** Please submit only one request per issue to keep discussions focused and manageable. diff --git a/vendor/fish-speech/.github/workflows/build-docker-image.yml b/vendor/fish-speech/.github/workflows/build-docker-image.yml new file mode 100644 index 0000000000000000000000000000000000000000..5e19ba6c0cd9d85b437400c0462454f99cab1267 --- /dev/null +++ b/vendor/fish-speech/.github/workflows/build-docker-image.yml @@ -0,0 +1,78 @@ +name: Build Docker Images + +on: + push: + branches: + - main + tags: + - "v*" + +jobs: + build: + runs-on: ubuntu-latest-16c64g + strategy: + matrix: + target: [webui, server] + backend: [cuda, cpu] + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Get Version + run: | + if [[ $GITHUB_REF == refs/tags/v* ]]; then + version=$(basename ${GITHUB_REF}) + else + version=nightly + fi + echo "version=${version}" >> $GITHUB_ENV + echo "Current version: ${version}" + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USER }} + password: ${{ secrets.DOCKER_PAT }} + + - name: Set platform for CPU builds + id: platform + run: | + if [ "${{ matrix.backend }}" = "cpu" ]; then + echo "platforms=linux/amd64,linux/arm64" >> $GITHUB_OUTPUT + else + echo "platforms=linux/amd64" >> $GITHUB_OUTPUT + fi + + - name: Build and Push ${{ matrix.target }}-${{ matrix.backend }} Image + uses: docker/build-push-action@v6 + with: + context: . + file: docker/Dockerfile + platforms: ${{ steps.platform.outputs.platforms }} + push: true + target: ${{ matrix.target }} + build-args: | + BACKEND=${{ matrix.backend }} + UV_EXTRA=${{ matrix.backend == 'cuda' && 'cu126' || 'cpu' }} + tags: | + fishaudio/fish-speech:${{ matrix.target }}-${{ matrix.backend }}-${{ env.version }} + fishaudio/fish-speech:${{ matrix.target }}-${{ matrix.backend }} + ${{ (matrix.target == 'webui' && matrix.backend == 'cuda') && format('fishaudio/fish-speech:{0}', env.version) || '' }} + ${{ (matrix.target == 'webui' && matrix.backend == 'cuda') && 'fishaudio/fish-speech:latest' || '' }} + outputs: type=image,oci-mediatypes=true,compression=zstd,compression-level=3,force-compression=true + cache-from: type=registry,ref=fishaudio/fish-speech:${{ matrix.target }}-${{ matrix.backend }} + cache-to: type=inline + + update-readme: + runs-on: ubuntu-latest + needs: build + if: github.ref == 'refs/heads/main' + steps: + - name: Push README to Dockerhub + uses: peter-evans/dockerhub-description@v4 + with: + username: ${{ secrets.DOCKER_USER }} + password: ${{ secrets.DOCKER_PAT }} + repository: fishaudio/fish-speech diff --git a/vendor/fish-speech/.github/workflows/docs.yml b/vendor/fish-speech/.github/workflows/docs.yml new file mode 100644 index 0000000000000000000000000000000000000000..0967ec0b00c3e1c0392f1b54d9d6200b18c00b46 --- /dev/null +++ b/vendor/fish-speech/.github/workflows/docs.yml @@ -0,0 +1,33 @@ +name: docs +on: + push: + branches: + - main + paths: + - 'docs/**' + - 'mkdocs.yml' + +permissions: + contents: write + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Configure Git Credentials + run: | + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + - uses: actions/setup-python@v5 + with: + python-version: 3.x + - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV + - uses: actions/cache@v4 + with: + key: mkdocs-material-${{ env.cache_id }} + path: .cache + restore-keys: | + mkdocs-material- + - run: pip install -r docs/requirements.txt + - run: mkdocs gh-deploy --force diff --git a/vendor/fish-speech/.github/workflows/stale.yml b/vendor/fish-speech/.github/workflows/stale.yml new file mode 100644 index 0000000000000000000000000000000000000000..47f4405c3309162333698f352d8012e2eee5d48c --- /dev/null +++ b/vendor/fish-speech/.github/workflows/stale.yml @@ -0,0 +1,25 @@ +name: Close inactive issues +on: + schedule: + - cron: "0 0 * * *" + +jobs: + close-issues: + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + steps: + - uses: actions/stale@v9 + with: + days-before-issue-stale: 30 + days-before-issue-close: 14 + stale-issue-label: "stale" + stale-issue-message: "This issue is stale because it has been open for 30 days with no activity." + close-issue-message: "This issue was closed because it has been inactive for 14 days since being marked as stale." + days-before-pr-stale: 30 + days-before-pr-close: 30 + stale-pr-label: "stale" + stale-pr-message: "This PR is stale because it has been open for 30 days with no activity." + close-pr-message: "This PR was closed because it has been inactive for 30 days since being marked as stale." + repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/vendor/fish-speech/awesome_webui/public/vite.svg b/vendor/fish-speech/awesome_webui/public/vite.svg new file mode 100644 index 0000000000000000000000000000000000000000..ee9fadaf9c4a762ac0ec010ca16ce8fa39a09e56 --- /dev/null +++ b/vendor/fish-speech/awesome_webui/public/vite.svg @@ -0,0 +1 @@ + diff --git a/vendor/fish-speech/awesome_webui/src/App.tsx b/vendor/fish-speech/awesome_webui/src/App.tsx new file mode 100644 index 0000000000000000000000000000000000000000..0e8b96fbd77a66eb46ccf0ae7d1347143d2e717f --- /dev/null +++ b/vendor/fish-speech/awesome_webui/src/App.tsx @@ -0,0 +1,1185 @@ +import { useEffect, useRef, useState } from 'react' +import { + AudioLines, + ChevronDown, + CircleAlert, + Copy, + Download, + FileText, + Info, + LoaderCircle, + Plus, + Settings2, + Upload, +} from 'lucide-react' + +import { Alert, AlertDescription, AlertTitle } from '@/components/ui/alert' +import { Badge } from '@/components/ui/badge' +import { Button } from '@/components/ui/button' +import { + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from '@/components/ui/card' +import { + Collapsible, + CollapsibleContent, + CollapsibleTrigger, +} from '@/components/ui/collapsible' +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from '@/components/ui/dialog' +import { Label } from '@/components/ui/label' +import { ScrollArea } from '@/components/ui/scroll-area' +import { Separator } from '@/components/ui/separator' +import { Slider } from '@/components/ui/slider' +import { Switch } from '@/components/ui/switch' +import { Textarea } from '@/components/ui/textarea' +import { ToggleGroup, ToggleGroupItem } from '@/components/ui/toggle-group' + +type AudioFormat = 'mp3' | 'wav' | 'pcm' | 'opus' +type LatencyMode = 'normal' | 'balanced' + +const defaultInputText = `[excited, joyful tone] We're going to DISNEY WORLD! [squeal of delight] I've been saving for [emphasis] three years [breathless] and finally, FINALLY we can go! The look on your face right now is worth every extra shift I worked! +[angry] After everything we've been through [break] I can't believe you would [emphasize] betray me like this. I gave you EVERYTHING! And now I'm left with nothing but memories and broken promises!` + + +type ControlsState = { + chunkLength: number + maxNewTokens: number + temperature: number + topP: number + repetitionPenalty: number + normalize: boolean + format: AudioFormat + latency: LatencyMode +} + +type Metrics = { + textLength: number + ttftMs: number + receivedKb: number +} + +type StatusState = { + tone: 'error' | 'info' + message: string +} + +type ReferenceItem = { + id: number + name: string + audio: ArrayBuffer + text: string + previewUrl: string +} + +type SpeakerGroup = { + id: number + references: ReferenceItem[] +} + +type PendingReference = { + mode: 'create' | 'edit' + speakerId: number + referenceId?: number + name: string + audio?: ArrayBuffer + text: string +} + +const initialControls: ControlsState = { + chunkLength: 1000, + maxNewTokens: 2048, + temperature: 0.9, + topP: 0.9, + repetitionPenalty: 1.05, + normalize: false, + format: 'mp3', + latency: 'normal', +} + +const formatMimeMap: Record = { + mp3: 'audio/mpeg', + wav: 'audio/wav', + pcm: 'audio/pcm', + opus: 'audio/opus', +} + +function createId() { + return Date.now() + Math.floor(Math.random() * 100000) +} + +function arrayBufferToBase64(buffer: ArrayBuffer): string { + const bytes = new Uint8Array(buffer) + let binary = '' + for (let i = 0; i < bytes.byteLength; i++) { + binary += String.fromCharCode(bytes[i]) + } + return btoa(binary) +} + +function createSpeakerGroup(): SpeakerGroup { + return { + id: createId(), + references: [], + } +} + +const initialSpeakerGroup = createSpeakerGroup() + +function buildReferencesPayload( + speakerGroups: SpeakerGroup[], + includeBinaryAudio: boolean, +) { + return speakerGroups.flatMap((speakerGroup) => + speakerGroup.references.map((reference) => ({ + text: reference.text, + audio: includeBinaryAudio + ? arrayBufferToBase64(reference.audio) + : '