Deploy MuQ-MuLan music-text similarity endpoint

.gitignore

@@ -0,0 +1,6 @@
+.venv/
+.ruff_cache/
+__pycache__/
+*.py[cod]
+.DS_Store
+outputs/

README.md

@@ -1,15 +1,49 @@
 ---
-title: MuQ MuLan
-emoji: 👀
-colorFrom: red
-colorTo: gray
+title: MuQ-MuLan
+emoji: 🎧
+colorFrom: blue
+colorTo: green
 sdk: gradio
-sdk_version: 6.20.0
-python_version: '3.13'
+sdk_version: 5.28.0
+python_version: '3.12'
 app_file: app.py
 pinned: false
 license: cc-by-nc-4.0
-short_description: Music-text similarity for music clips
+short_description: Rank music descriptions against an audio clip
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# MuQ-MuLan Music-Text Similarity
+
+A HARP-compatible deployment of MuQ-MuLan for comparing a music clip with
+candidate text descriptions. It can support workflows such as tagging,
+searching a sample library, or choosing the description that best matches a
+piece of music.
+
+The Space loads the official MuQ-MuLan-large checkpoint from Hugging Face.
+Model files are downloaded at runtime and cached by the Space.
+
+## Inputs
+
+- One music audio clip between 10 and 60 seconds
+- One to eight candidate descriptions, one per line
+
+Audio is converted to mono and resampled to 24 kHz. MuQ-MuLan processes
+10-second windows and averages their embeddings for longer clips. Text may be
+written in English or Chinese.
+
+## Output
+
+A JSON file containing the candidate descriptions in descending similarity
+order. Scores are cosine similarities from -1 to 1, not calibrated
+probabilities.
+
+## Sources
+
+See [SOURCES.md](SOURCES.md) for the model, source revision, license, and
+associated paper.
+
+## License
+
+The upstream source code is MIT licensed. The MuQ-MuLan model weights are
+released under CC BY-NC 4.0, so this deployment is intended for
+non-commercial use.

SOURCES.md

@@ -0,0 +1,30 @@
+# Sources
+
+## MuQ and MuQ-MuLan
+
+- Paper: https://arxiv.org/abs/2501.01108
+- Source: https://github.com/tencent-ailab/MuQ
+- Pinned source revision:
+  `28847ea50cd31ac4b8b6a7dacc051ad7d1c7606a`
+- Source license: MIT
+- Checkpoint: https://huggingface.co/OpenMuQ/MuQ-MuLan-large
+- Pinned checkpoint revision:
+  `2e01c796b71dca71b45251384c04cd7b237c9020`
+- Checkpoint license: CC BY-NC 4.0
+
+MuQ-MuLan is a joint music-text embedding model trained through contrastive
+learning. Its audio and text embeddings are L2-normalized, so their dot
+product is cosine similarity. The official model supports English and Chinese
+text. For audio longer than 10 seconds, the official inference code embeds
+non-overlapping 10-second windows and averages their representations.
+
+## Encoders
+
+- Audio encoder: https://huggingface.co/OpenMuQ/MuQ-large-msd-iter
+- Pinned audio revision:
+  `0562a57814f6f8bbd9fdea0a25921a2fce1a841a`
+- Audio model license: CC BY-NC 4.0
+- Text encoder: https://huggingface.co/FacebookAI/xlm-roberta-base
+- Pinned text revision:
+  `e73636d4f797dec63c3081bb6ed5c7b0bb3f2089`
+- Text model license: MIT

app.py

@@ -0,0 +1,164 @@
+from __future__ import annotations
+
+import json
+import tempfile
+import uuid
+from pathlib import Path
+
+import gradio as gr
+import soundfile as sf
+
+try:
+    import spaces
+except ImportError:
+    class spaces:
+        class GPU:
+            def __init__(self, func=None, duration=60):
+                self.func = func
+
+            def __call__(self, *args, **kwargs):
+                if self.func is not None:
+                    return self.func(*args, **kwargs)
+                return args[0]
+
+from pyharp import ModelCard, build_endpoint
+
+from muq_mulan_runtime import rank_descriptions
+
+MIN_AUDIO_SECONDS = 10
+MAX_AUDIO_SECONDS = 60
+MAX_DESCRIPTIONS = 8
+MAX_DESCRIPTION_LENGTH = 300
+OUTPUT_ROOT = Path(tempfile.gettempdir()) / "muq_mulan_outputs"
+
+model_card = ModelCard(
+    name="MuQ-MuLan",
+    description=(
+        "Rank English or Chinese music descriptions by their similarity "
+        "to an uploaded music clip."
+    ),
+    author="Tencent AI Lab",
+    tags=[
+        "music-information-retrieval",
+        "music-text-retrieval",
+        "music-tagging",
+        "audio-analysis",
+    ],
+)
+
+
+def _validate_audio(path: str | None) -> str:
+    if not path:
+        raise gr.Error("Please upload a music clip.")
+
+    try:
+        duration = sf.info(path).duration
+    except Exception as exc:
+        raise gr.Error(f"Could not read the audio file: {exc}") from exc
+
+    if duration < MIN_AUDIO_SECONDS:
+        raise gr.Error(
+            f"Audio must be at least {MIN_AUDIO_SECONDS} seconds long. "
+            f"Received {duration:.1f} seconds."
+        )
+    if duration > MAX_AUDIO_SECONDS:
+        raise gr.Error(
+            f"Audio must be no longer than {MAX_AUDIO_SECONDS} seconds. "
+            f"Received {duration:.1f} seconds."
+        )
+    return path
+
+
+def _parse_descriptions(value: str | None) -> list[str]:
+    descriptions = [
+        line.strip()
+        for line in (value or "").splitlines()
+        if line.strip()
+    ]
+    if not descriptions:
+        raise gr.Error("Enter at least one music description.")
+    if len(descriptions) > MAX_DESCRIPTIONS:
+        raise gr.Error(
+            f"Enter no more than {MAX_DESCRIPTIONS} descriptions."
+        )
+    if any(len(description) > MAX_DESCRIPTION_LENGTH for description in descriptions):
+        raise gr.Error(
+            "Each description must be no more than "
+            f"{MAX_DESCRIPTION_LENGTH} characters."
+        )
+    return descriptions
+
+
+@spaces.GPU(duration=240)
+def process_fn(
+    input_audio: str | None,
+    candidate_descriptions: str | None,
+) -> str:
+    input_audio = _validate_audio(input_audio)
+    descriptions = _parse_descriptions(candidate_descriptions)
+
+    try:
+        results = rank_descriptions(input_audio, descriptions)
+    except Exception as exc:
+        raise gr.Error(f"MuQ-MuLan inference failed: {exc}") from exc
+
+    output_dir = OUTPUT_ROOT / uuid.uuid4().hex
+    output_dir.mkdir(parents=True, exist_ok=True)
+    output_path = output_dir / "muq_mulan_similarity.json"
+    output_path.write_text(
+        json.dumps(
+            {
+                "model": "OpenMuQ/MuQ-MuLan-large",
+                "score_type": "cosine_similarity",
+                "score_range": [-1.0, 1.0],
+                "results": results,
+            },
+            ensure_ascii=False,
+            indent=2,
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+    return str(output_path)
+
+
+with gr.Blocks(title="MuQ-MuLan Music-Text Similarity") as demo:
+    input_components = [
+        gr.Audio(
+            type="filepath",
+            label="Music Audio",
+        )
+        .harp_required(True)
+        .set_info("Music clip between 10 and 60 seconds long."),
+        gr.Textbox(
+            lines=5,
+            label="Candidate Descriptions",
+            placeholder=(
+                "upbeat electronic dance music\n"
+                "slow acoustic ballad\n"
+                "一首轻快的钢琴曲"
+            ),
+        )
+        .harp_required(True)
+        .set_info("Enter one English or Chinese description per line."),
+    ]
+    output_components = [
+        gr.File(
+            type="filepath",
+            file_types=[".json"],
+            label="Similarity Ranking",
+        ).set_info("Descriptions ranked by cosine similarity."),
+    ]
+    build_endpoint(
+        model_card=model_card,
+        input_components=input_components,
+        output_components=output_components,
+        process_fn=process_fn,
+    )
+
+
+if __name__ == "__main__":
+    demo.queue(default_concurrency_limit=1).launch(
+        show_error=True,
+        pwa=True,
+    )

model.json

@@ -0,0 +1,41 @@
+{
+  "name": "MuQ-MuLan",
+  "task": "Music-text similarity and description ranking",
+  "source_repository": "tencent-ailab/MuQ",
+  "source_revision": "28847ea50cd31ac4b8b6a7dacc051ad7d1c7606a",
+  "source_license": "MIT",
+  "checkpoint": {
+    "repo": "OpenMuQ/MuQ-MuLan-large",
+    "revision": "2e01c796b71dca71b45251384c04cd7b237c9020",
+    "license": "CC BY-NC 4.0"
+  },
+  "audio_encoder": {
+    "repo": "OpenMuQ/MuQ-large-msd-iter",
+    "revision": "0562a57814f6f8bbd9fdea0a25921a2fce1a841a",
+    "license": "CC BY-NC 4.0"
+  },
+  "text_encoder": {
+    "repo": "FacebookAI/xlm-roberta-base",
+    "revision": "e73636d4f797dec63c3081bb6ed5c7b0bb3f2089",
+    "license": "MIT"
+  },
+  "input": {
+    "sample_rate": 24000,
+    "channels": 1,
+    "minimum_duration_seconds": 10,
+    "maximum_duration_seconds": 60,
+    "window_seconds": 10,
+    "maximum_descriptions": 8,
+    "text_languages": [
+      "English",
+      "Chinese"
+    ]
+  },
+  "output": {
+    "type": "cosine_similarity_ranking",
+    "range": [
+      -1.0,
+      1.0
+    ]
+  }
+}

muq_mulan_runtime.py

@@ -0,0 +1,110 @@
+from __future__ import annotations
+
+import json
+from functools import lru_cache
+from pathlib import Path
+
+import soundfile as sf
+import torch
+import torchaudio.functional as AF
+from huggingface_hub import hf_hub_download, snapshot_download
+from muq import MuQMuLan
+
+SAMPLE_RATE = 24_000
+MODEL_REPO = "OpenMuQ/MuQ-MuLan-large"
+MODEL_REVISION = "2e01c796b71dca71b45251384c04cd7b237c9020"
+AUDIO_MODEL_REPO = "OpenMuQ/MuQ-large-msd-iter"
+AUDIO_MODEL_REVISION = "0562a57814f6f8bbd9fdea0a25921a2fce1a841a"
+TEXT_MODEL_REPO = "xlm-roberta-base"
+TEXT_MODEL_REVISION = "e73636d4f797dec63c3081bb6ed5c7b0bb3f2089"
+
+
+def _select_device() -> str:
+    if torch.cuda.is_available():
+        return "cuda"
+    if torch.backends.mps.is_available():
+        return "mps"
+    return "cpu"
+
+
+@lru_cache(maxsize=1)
+def _load_model() -> MuQMuLan:
+    config_path = hf_hub_download(
+        repo_id=MODEL_REPO,
+        filename="config.json",
+        revision=MODEL_REVISION,
+    )
+    config = json.loads(Path(config_path).read_text(encoding="utf-8"))
+
+    audio_model_path = snapshot_download(
+        repo_id=AUDIO_MODEL_REPO,
+        revision=AUDIO_MODEL_REVISION,
+        allow_patterns=[
+            "config.json",
+            "model.safetensors",
+        ],
+    )
+    text_model_path = snapshot_download(
+        repo_id=TEXT_MODEL_REPO,
+        revision=TEXT_MODEL_REVISION,
+        allow_patterns=[
+            "config.json",
+            "model.safetensors",
+            "sentencepiece.bpe.model",
+            "special_tokens_map.json",
+            "tokenizer.json",
+            "tokenizer_config.json",
+        ],
+    )
+    config["audio_model"]["name"] = audio_model_path
+    config["text_model"]["name"] = text_model_path
+
+    return MuQMuLan.from_pretrained(
+        MODEL_REPO,
+        revision=MODEL_REVISION,
+        config=config,
+    ).eval()
+
+
+def _load_audio(path: str) -> torch.Tensor:
+    audio, sample_rate = sf.read(
+        Path(path),
+        dtype="float32",
+        always_2d=True,
+    )
+    waveform = torch.from_numpy(audio).mean(dim=1)
+    if sample_rate != SAMPLE_RATE:
+        waveform = AF.resample(waveform, sample_rate, SAMPLE_RATE)
+    return waveform.unsqueeze(0)
+
+
+@torch.inference_mode()
+def rank_descriptions(
+    audio_path: str,
+    descriptions: list[str],
+) -> list[dict[str, object]]:
+    device_name = _select_device()
+    device = torch.device(device_name)
+    model = _load_model().to(device)
+    waveform = _load_audio(audio_path).to(device)
+
+    audio_embedding = model(wavs=waveform)
+    text_embeddings = model(texts=descriptions)
+    scores = model.calc_similarity(
+        audio_embedding,
+        text_embeddings,
+    )[0].detach().cpu().tolist()
+
+    ranked = sorted(
+        zip(descriptions, scores),
+        key=lambda item: item[1],
+        reverse=True,
+    )
+    return [
+        {
+            "rank": index,
+            "description": description,
+            "similarity": round(float(score), 6),
+        }
+        for index, (description, score) in enumerate(ranked, start=1)
+    ]

requirements.txt

@@ -0,0 +1,9 @@
+gradio==5.28.0
+torch==2.8.0
+torchaudio==2.8.0
+transformers==4.53.3
+huggingface-hub>=0.33,<1
+numpy>=1.26,<3
+soundfile>=0.12.1,<1
+git+https://github.com/tencent-ailab/MuQ.git@28847ea50cd31ac4b8b6a7dacc051ad7d1c7606a
+git+https://github.com/TEAMuP-dev/pyharp.git@d65c4f7d0264dcdb3024a6c5466cddd7b2defdca