lance_mlx/__init__.py

"""Lance MLX — first MLX port of ByteDance's unified multimodal model.

Status (2026-05-19):
  modeling_utils      — DONE, parity tested
  vae_wan22           — image mode DONE; video streaming cache PENDING
  lance.py            — backbone + adapter + sampler primitives DONE
  load_lance helper   — loads a converted bundle in one call
"""
from __future__ import annotations

import inspect
import json
from pathlib import Path
from typing import Tuple

import mlx.core as mx

from . import modeling_utils, vae_wan22, lance  # noqa: F401
from .lance import Lance, LanceConfig

__version__ = "0.0.2"


def load_lance(bundle_dir: str | Path) -> Tuple[Lance, dict]:
    """Build Lance from a converted bundle directory and load all weights.

    Expected files in `bundle_dir`:
      config.json
      model.safetensors
      vit.safetensors   (optional — needed for understanding / image-conditioned)
      vae.safetensors   (optional — needed for encode/decode latent <-> pixel)

    Returns (model, cfg_dict).
    """
    from mlx_vlm.models.qwen2_5_vl.config import (
        ModelConfig as Qwen25VLConfig,
        TextConfig,
        VisionConfig,
    )

    bundle_dir = Path(bundle_dir)
    cfg = json.loads((bundle_dir / "config.json").read_text())
    qcfg_dict = cfg["qwen2_5_vl_config"]

    text_sig = inspect.signature(TextConfig).parameters
    text_params = {
        k: v for k, v in qcfg_dict.items()
        if k != "vision_config" and k in text_sig
    }
    text_cfg = TextConfig(**text_params)

    vision_sig = inspect.signature(VisionConfig).parameters
    vision_params = {
        k: v for k, v in qcfg_dict["vision_config"].items()
        if k in vision_sig
    }
    vision_cfg = VisionConfig(**vision_params)

    model_sig = inspect.signature(Qwen25VLConfig).parameters
    model_params = {k: v for k, v in qcfg_dict.items() if k in model_sig}
    model_params["text_config"] = text_cfg
    model_params["vision_config"] = vision_cfg
    qcfg = Qwen25VLConfig(**model_params)

    lance_kwargs = {k: v for k, v in cfg.items() if k != "qwen2_5_vl_config"
                    and k in inspect.signature(LanceConfig).parameters}
    if "latent_patch_size" in lance_kwargs and isinstance(
        lance_kwargs["latent_patch_size"], list
    ):
        lance_kwargs["latent_patch_size"] = tuple(lance_kwargs["latent_patch_size"])
    model = Lance(LanceConfig(qwen_config=qcfg, **lance_kwargs))

    # Bundle key prefixes are Qwen2.5-VL convention (language_model.*, vision_tower.*)
    # plus Lance adapter top-levels (vae2llm, llm2vae, time_embedder, latent_pos_embed)
    # plus vae_model.* for the Wan VAE shards. Our wrapper stores Qwen inside
    # self.backbone, so language_model/vision_tower must be prefixed with `backbone.`.
    # The VAE lives outside the Lance class (separate module), so vae_model.* keys
    # are filtered here and returned via cfg['_vae_weights'] for the caller.
    raw_weights: dict[str, mx.array] = {}
    for fname in ("model.safetensors", "vit.safetensors", "vae.safetensors"):
        path = bundle_dir / fname
        if path.exists():
            raw_weights.update(mx.load(str(path)))

    lance_weights: dict[str, mx.array] = {}
    vae_weights: dict[str, mx.array] = {}
    moe_gen_weights: dict[str, mx.array] = {}
    for k, v in raw_weights.items():
        if k.startswith("vae_model."):
            vae_weights[k[len("vae_model."):]] = v
        elif "_moe_gen" in k:
            # Lance's generation-path weights — not consumed by the bare-Qwen
            # backbone in this wrapper. Returned separately so a future
            # MoE-aware port can pick them up.
            moe_gen_weights[k] = v
        elif k.startswith("language_model.") or k.startswith("vision_tower."):
            lance_weights["backbone." + k] = v
        else:
            lance_weights[k] = v

    model.load_weights(list(lance_weights.items()), strict=False)
    cfg["_vae_weights"] = vae_weights
    cfg["_moe_gen_weights"] = moe_gen_weights
    cfg["_loaded_into_model"] = len(lance_weights)
    return model, cfg