Spaces:
Running on Zero
Running on Zero
fix
Browse files
app.py
CHANGED
|
@@ -17,7 +17,10 @@ import hashlib
|
|
| 17 |
from slugify import slugify
|
| 18 |
import argparse
|
| 19 |
import importlib
|
|
|
|
|
|
|
| 20 |
import sys
|
|
|
|
| 21 |
from pathlib import Path
|
| 22 |
import tempfile
|
| 23 |
import zipfile
|
|
@@ -32,6 +35,7 @@ training_script_url = f"https://raw.githubusercontent.com/huggingface/diffusers/
|
|
| 32 |
orchestrator_script_url = "https://huggingface.co/datasets/multimodalart/lora-ease-helper/raw/main/script.py"
|
| 33 |
|
| 34 |
MAGE_VL_MODEL_ID = "microsoft/Mage-VL"
|
|
|
|
| 35 |
mage_vl_processor = None
|
| 36 |
mage_vl_model = None
|
| 37 |
caption_cache = {}
|
|
@@ -73,13 +77,17 @@ def get_captioner():
|
|
| 73 |
"""Load the embedded Mage-VL model once per Space process."""
|
| 74 |
global mage_vl_processor, mage_vl_model
|
| 75 |
if mage_vl_processor is None or mage_vl_model is None:
|
|
|
|
| 76 |
target_device = "cuda" if is_spaces or torch.cuda.is_available() else "cpu"
|
| 77 |
dtype = torch.bfloat16 if target_device == "cuda" else torch.float32
|
| 78 |
mage_vl_processor = AutoProcessor.from_pretrained(
|
| 79 |
-
MAGE_VL_MODEL_ID,
|
|
|
|
|
|
|
| 80 |
)
|
| 81 |
mage_vl_model = AutoModelForCausalLM.from_pretrained(
|
| 82 |
MAGE_VL_MODEL_ID,
|
|
|
|
| 83 |
trust_remote_code=True,
|
| 84 |
dtype=dtype,
|
| 85 |
attn_implementation="sdpa",
|
|
@@ -87,6 +95,41 @@ def get_captioner():
|
|
| 87 |
return mage_vl_processor, mage_vl_model
|
| 88 |
|
| 89 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 90 |
def _mage_prompt(processor, instruction):
|
| 91 |
messages = [
|
| 92 |
{
|
|
|
|
| 17 |
from slugify import slugify
|
| 18 |
import argparse
|
| 19 |
import importlib
|
| 20 |
+
import importlib.machinery
|
| 21 |
+
import importlib.util
|
| 22 |
import sys
|
| 23 |
+
import types
|
| 24 |
from pathlib import Path
|
| 25 |
import tempfile
|
| 26 |
import zipfile
|
|
|
|
| 35 |
orchestrator_script_url = "https://huggingface.co/datasets/multimodalart/lora-ease-helper/raw/main/script.py"
|
| 36 |
|
| 37 |
MAGE_VL_MODEL_ID = "microsoft/Mage-VL"
|
| 38 |
+
MAGE_VL_REVISION = "d88b153285f1633a61b2f693c59c8576693af185"
|
| 39 |
mage_vl_processor = None
|
| 40 |
mage_vl_model = None
|
| 41 |
caption_cache = {}
|
|
|
|
| 77 |
"""Load the embedded Mage-VL model once per Space process."""
|
| 78 |
global mage_vl_processor, mage_vl_model
|
| 79 |
if mage_vl_processor is None or mage_vl_model is None:
|
| 80 |
+
_install_image_only_mamba_shim()
|
| 81 |
target_device = "cuda" if is_spaces or torch.cuda.is_available() else "cpu"
|
| 82 |
dtype = torch.bfloat16 if target_device == "cuda" else torch.float32
|
| 83 |
mage_vl_processor = AutoProcessor.from_pretrained(
|
| 84 |
+
MAGE_VL_MODEL_ID,
|
| 85 |
+
revision=MAGE_VL_REVISION,
|
| 86 |
+
trust_remote_code=True,
|
| 87 |
)
|
| 88 |
mage_vl_model = AutoModelForCausalLM.from_pretrained(
|
| 89 |
MAGE_VL_MODEL_ID,
|
| 90 |
+
revision=MAGE_VL_REVISION,
|
| 91 |
trust_remote_code=True,
|
| 92 |
dtype=dtype,
|
| 93 |
attn_implementation="sdpa",
|
|
|
|
| 95 |
return mage_vl_processor, mage_vl_model
|
| 96 |
|
| 97 |
|
| 98 |
+
def _install_image_only_mamba_shim():
|
| 99 |
+
"""Satisfy Mage-VL's optional StreamMind import without compiling Mamba.
|
| 100 |
+
|
| 101 |
+
Transformers scans every relative remote-code file, including the lazily
|
| 102 |
+
loaded StreamMind gate. Still-image captioning never constructs that gate,
|
| 103 |
+
so requiring its CUDA extension would add a large, unnecessary build step.
|
| 104 |
+
"""
|
| 105 |
+
if importlib.util.find_spec("mamba_ssm") is not None:
|
| 106 |
+
return
|
| 107 |
+
|
| 108 |
+
def module(name, package=False):
|
| 109 |
+
value = types.ModuleType(name)
|
| 110 |
+
value.__spec__ = importlib.machinery.ModuleSpec(
|
| 111 |
+
name, loader=None, is_package=package
|
| 112 |
+
)
|
| 113 |
+
if package:
|
| 114 |
+
value.__path__ = []
|
| 115 |
+
sys.modules[name] = value
|
| 116 |
+
return value
|
| 117 |
+
|
| 118 |
+
mamba_module = module("mamba_ssm", package=True)
|
| 119 |
+
models_module = module("mamba_ssm.models", package=True)
|
| 120 |
+
mixer_module = module("mamba_ssm.models.mixer_seq_simple")
|
| 121 |
+
|
| 122 |
+
def unavailable_create_block(*args, **kwargs):
|
| 123 |
+
raise RuntimeError(
|
| 124 |
+
"Mage-VL's optional StreamMind gate requires mamba-ssm; this Space "
|
| 125 |
+
"embeds Mage-VL for still-image captioning only."
|
| 126 |
+
)
|
| 127 |
+
|
| 128 |
+
mixer_module.create_block = unavailable_create_block
|
| 129 |
+
mamba_module.models = models_module
|
| 130 |
+
models_module.mixer_seq_simple = mixer_module
|
| 131 |
+
|
| 132 |
+
|
| 133 |
def _mage_prompt(processor, instruction):
|
| 134 |
messages = [
|
| 135 |
{
|