Text Generation
Transformers
Safetensors
English
Chinese
Russian
yue2
music-generation
orbitquant
quantization
4-bit precision
custom-code
8-bit precision
Instructions to use WaveCut/YuE2-3B-OrbitQuant-W4A4 with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use WaveCut/YuE2-3B-OrbitQuant-W4A4 with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-generation", model="WaveCut/YuE2-3B-OrbitQuant-W4A4")# Load model directly from transformers import AutoModelForCausalLM model = AutoModelForCausalLM.from_pretrained("WaveCut/YuE2-3B-OrbitQuant-W4A4", device_map="auto") - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use WaveCut/YuE2-3B-OrbitQuant-W4A4 with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "WaveCut/YuE2-3B-OrbitQuant-W4A4" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "WaveCut/YuE2-3B-OrbitQuant-W4A4", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }'Use Docker
docker model run hf.co/WaveCut/YuE2-3B-OrbitQuant-W4A4
- SGLang
How to use WaveCut/YuE2-3B-OrbitQuant-W4A4 with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "WaveCut/YuE2-3B-OrbitQuant-W4A4" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "WaveCut/YuE2-3B-OrbitQuant-W4A4", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "WaveCut/YuE2-3B-OrbitQuant-W4A4" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "WaveCut/YuE2-3B-OrbitQuant-W4A4", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }' - Docker Model Runner
How to use WaveCut/YuE2-3B-OrbitQuant-W4A4 with Docker Model Runner:
docker model run hf.co/WaveCut/YuE2-3B-OrbitQuant-W4A4
| { | |
| "runtime_version": "10", | |
| "orbitquant_version": "0.9.7", | |
| "orbitquant_commit": "5c624b088cd9378a135a5fe44bc4ea71f5d288da", | |
| "native_kernel_version": "1.0.4", | |
| "native_wheel_sha256": "13aa3db06ae9e90d9c9ef90ed3ead367743bd8582dc8731c82c5e35d47b5ae16", | |
| "fast_kernel_source_commit": "7795243b1b9066b5463b4625f16e36e6e4e6682f", | |
| "fast_kernel_build_note": "kernel-builder generated files made the build namespace dirty; source content and exact binary are included and hashed", | |
| "experimental_archive": { | |
| "repo": "WaveCut/YuE2-3B-OrbitQuant-lab-private", | |
| "revision": "1edba493f504208488fd7d98fe1312ff52b02365", | |
| "path": "profile-round2" | |
| }, | |
| "weight_generation_source_lock": "../source-lock.json", | |
| "orbitquant_wheel_sha256": "7f612c79441b14529c9a7d0f35cb22cc871e3a9b33d24f9a19352cb3e2ff9bdb", | |
| "rms_quant_kernel": { | |
| "source_commit": "92dbd686220262d8d24b09c27a6d6703e19f9465", | |
| "module": "_yue2_qkv_fused_cuda_cf0db58", | |
| "binary_sha256": "1ca79342c379a231d86fa5c8e94e45134c3624d7b85ca5e55997004ec69ff542", | |
| "source_path": "kernel-source/rms-quant", | |
| "arithmetic": "FP32 x*gamma; RMS scalar folded into activation norm; original epsilon rescaled before flooring", | |
| "scope": "explicit fuse_rms_quant flag; AR decode only; eager prefill and NAR retain existing paths" | |
| }, | |
| "round5_archive": { | |
| "repo": "WaveCut/YuE2-3B-OrbitQuant-lab-private", | |
| "revision": "ab852c01399038009c9db216f16cd22cd4390f82", | |
| "path": "profile-round5" | |
| }, | |
| "captured_sampling": { | |
| "scope": "request-owned combined decode/CFG/sampling graph for ABC and semantic; fast/memory; positive temperature and CUDA graph decode only", | |
| "rng": "caller-owned generator registered with CUDA graph; capture/warmup restore advanced state; no module cache", | |
| "opt_out": "--no-captured-sampling / capture_sampling=False", | |
| "research_archive": { | |
| "repo": "WaveCut/YuE2-3B-OrbitQuant-lab-private", | |
| "revision": "acfb177adfb11dceb89fb8cc6b0382554c1c4e19", | |
| "path": "profile-round6/captured-sampler-interleaved" | |
| }, | |
| "abc_research_archive": { | |
| "repo": "WaveCut/YuE2-3B-OrbitQuant-lab-private", | |
| "revision": "b8892ba75691c8ab0427ebd96838bf78bec62a91", | |
| "path": "profile-round7/captured-abc-interleaved" | |
| }, | |
| "history": "device ring and counter; no per-token host history upload", | |
| "first_token": "separate prefill sampler; combined graph starts with the second token", | |
| "research_archive_v6": { | |
| "repo": "WaveCut/YuE2-3B-OrbitQuant-lab-private", | |
| "revision": "bdae011fa163b52e062f6f0855cddb7d4d1f73f0", | |
| "path": "profile-round8" | |
| } | |
| }, | |
| "paired_row_decode": { | |
| "rows": "2..8", | |
| "minimum_output_features": 2048, | |
| "input_features": "1024..16384", | |
| "weight_layout": "row-major", | |
| "archive_revision": "7344aedfa6c322229259796b3eab0941046f655e", | |
| "archive_path": "profile-round9/native-evidence", | |
| "native_build_workflow": 34665811056 | |
| }, | |
| "int8_kv_cache": { | |
| "source_commit": "fcfedee4713ca864d55641cf4902b4eeef725de3", | |
| "binary_sha256": "a7d62fa5d79d8f0e2d4bb6bd0676b1b59a8676c7fcb615785eadc230e1f88022", | |
| "source_path": "kernel-source/int8-kv", | |
| "scope": "optional AR decode cache; BF16 prefill; fast/memory with RMS activation fusion", | |
| "format": "symmetric signed INT8 per token/head, FP32 scales", | |
| "default": "bf16", | |
| "research_revision": "933fbe9dcc2c6eff697504593d0f5e93e2fff229", | |
| "profile_revision": "7f98a339b66c4c4cef76277f38c285c39027dca8" | |
| }, | |
| "register_codebook_decode": { | |
| "native_binary_sha256": "6fb8e94e30e2c9982da5e3e56c04dabed946fe9dc8ad1785c11cc3cf200217b4", | |
| "source_path": "kernel-source/orbitquant-native", | |
| "scope": "SM120 only; row-major M1..8, N>=2048, K1024..16384; arbitrary signed INT8 codebooks; paired-row reuse for M>=2", | |
| "native_build_workflow": 34674570012, | |
| "python_publish_workflow": 34675461162, | |
| "release_evidence": { | |
| "repo": "WaveCut/YuE2-3B-OrbitQuant-lab-private", | |
| "revision": "a88ed5684e6b7b3148b204d38052becb3d3a8ae8", | |
| "path": "profile-round12/published-release" | |
| } | |
| }, | |
| "int8_attention": { | |
| "default": "ordered", | |
| "layouts": { | |
| "ordered": { | |
| "source_commit": "193a8d7ad6951dc237435e988986e044bacfe7a9", | |
| "binary_sha256": "071c866680bb24506680a5d8b2b46c04f60e725ccf9ea6604e4e18973bca4385", | |
| "source_path": "kernel-source/int8-ordered" | |
| }, | |
| "subwarp": { | |
| "source_commit": "4d213ef482b4fbc2733a6e071bf9d5f07ebfa58b", | |
| "binary_sha256": "d12f0566a396a422eaa35515678fc142ba11b4338aef23c00adf9d095c823778", | |
| "source_path": "kernel-source/int8-subwarp" | |
| } | |
| }, | |
| "reference": "int8_kv_cache", | |
| "scope": "AR decode only; request GraphAR owns layout; BF16 prefill and cache format unchanged", | |
| "approximate_research": { | |
| "repo": "WaveCut/YuE2-3B-OrbitQuant-lab-private", | |
| "revision": "f4fc2828b6566f464393de29794f56fb1d68044e", | |
| "path": "profile-round14/subwarp-attention" | |
| }, | |
| "ordered_research": { | |
| "repo": "WaveCut/YuE2-3B-OrbitQuant-lab-private", | |
| "revision": "29502653219877cd2ab8d25466d9b2f39955ceb0", | |
| "path": "profile-round14/ordered-attention" | |
| } | |
| } | |
| } | |