fix the cohere transcriber geenrator : use HF quick start

crew2.py

@@ -61,37 +61,19 @@ except ImportError:
 ###### The agentic app
 
 # ------ LLM endpoint constants ------
+_VLLM_BASE_URL = "https://rcaz33--example-vllm-inference-serve.modal.run/v1"
 _VLLM_MODEL = "openai/google/gemma-4-26B-A4B-it"  # LiteLLM prefix — for crewai
 _VLLM_SERVED_MODEL = (
     "google/gemma-4-26B-A4B-it"  # actual vLLM served name — for direct API calls
 )
-_vllm_base_url = None
 
-
-def _get_vllm_base_url() -> str:
-    global _vllm_base_url
-    if _vllm_base_url is None:
-        import modal as _modal
-
-        _vllm_base_url = (
-            _modal.Function.from_name("vllm-inference", "serve").web_url + "/v1"
-        )
-    return _vllm_base_url
-
-
-_llm = None
-
-
-def _get_llm():
-    global _llm
-    if _llm is None:
-        _llm = LLM(
-            model=_VLLM_MODEL,
-            base_url=_get_vllm_base_url(),
-            api_key="sk-dummy-key-not-needed",
-            max_tokens=4096,
-        )
-    return _llm
+# Define our LLM using the Modal-deployed Gemma 4 26B model via vLLM (OpenAI-compatible API)
+llm = LLM(
+    model=_VLLM_MODEL,
+    base_url=_VLLM_BASE_URL,
+    api_key="sk-dummy-key-not-needed",
+    max_tokens=4096,
+)
 
 search_tool = SerperDevTool()
 
@@ -185,7 +167,7 @@ def run_pipeline(statement: str, session_id: str | None = None) -> dict:
             verbose=True,
             allow_delegation=False,
             tools=[search_tool],
-            llm=_get_llm(),
+            llm=llm,
             max_iter=1,
         )
 
@@ -215,7 +197,7 @@ def run_pipeline(statement: str, session_id: str | None = None) -> dict:
             verbose=True,
             allow_delegation=False,
             tools=[search_tool],
-            llm=_get_llm(),
+            llm=llm,
             max_iter=1,
         )
 
@@ -244,7 +226,7 @@ def run_pipeline(statement: str, session_id: str | None = None) -> dict:
             backstory="You are a world-class creative director who translates complex, contrasting ideas into powerful visual concepts.",
             verbose=True,
             allow_delegation=False,
-            llm=_get_llm(),
+            llm=llm,
             max_iter=1,
         )
 
@@ -383,7 +365,7 @@ VOICE_STYLE: <voice style description>""",
         )
         with _span_cm as _span:
             resp = _httpx.post(
-                f"{_get_vllm_base_url()}/chat/completions",
+                f"{_VLLM_BASE_URL}/chat/completions",
                 json=payload,
                 headers={"Authorization": "Bearer sk-dummy-key-not-needed"},
                 timeout=300,

transcribe_generator.py

@@ -11,8 +11,11 @@ transcribe_image = modal.Image.debian_slim(python_version="3.12").pip_install(
     "torch>=2.5.0",
     "transformers>=5.4.0",
     "huggingface_hub",
+    "soundfile",
+    "librosa",
     "fastapi[standard]",
     "requests",
+    "sentencepiece",
 )
 
 hf_cache_vol = modal.Volume.from_name("huggingface-cache", create_if_missing=True)
@@ -31,17 +34,45 @@ SAMPLE_RATE = 16000
 )
 class CohereTranscriber:
     def __init__(self):
-        from transformers import AutoProcessor, CohereAsrForConditionalGeneration
-
-        self.processor = AutoProcessor.from_pretrained(MODEL_NAME)
-        self.model = CohereAsrForConditionalGeneration.from_pretrained(
-            MODEL_NAME, device_map="auto"
+        import torch
+        from transformers import AutoProcessor
+
+        self.device = "cuda"
+        from transformers import AutoModelForSpeechSeq2Seq, AutoConfig
+        # Config: preprocessor.features=128, window_size=0.025s (400 @16kHz), window_stride=0.01s (160 @16kHz)
+        self.processor = AutoProcessor.from_pretrained(
+            MODEL_NAME,
+            trust_remote_code=True,
+            feature_size=128,
+            n_window_size=400,
+            n_window_stride=160,
+            n_fft=512,
         )
+        # Load config, get model class from pattern matching, patch list→set for transformers 5.12 compat
+        config = AutoConfig.from_pretrained(MODEL_NAME, trust_remote_code=True)
+        # Try loading; if it fails due to list|set, patch and retry
+        import transformers.modeling_utils as _mu
+        orig_fn = _mu.PreTrainedModel._adjust_missing_and_unexpected_keys
+        def patched_fn(self, *a, **kw):
+            if hasattr(self, '_keys_to_ignore_on_load_unexpected') and isinstance(self._keys_to_ignore_on_load_unexpected, list):
+                self._keys_to_ignore_on_load_unexpected = set(self._keys_to_ignore_on_load_unexpected)
+            return orig_fn(self, *a, **kw)
+        _mu.PreTrainedModel._adjust_missing_and_unexpected_keys = patched_fn
+        try:
+            self.model = AutoModelForSpeechSeq2Seq.from_pretrained(
+                MODEL_NAME,
+                dtype=torch.bfloat16,
+                trust_remote_code=True,
+            ).to(self.device)
+        finally:
+            _mu.PreTrainedModel._adjust_missing_and_unexpected_keys = orig_fn
 
     @modal.fastapi_endpoint(method="POST")
     def transcribe(self, body: dict) -> JSONResponse:
-        from transformers.audio_utils import load_audio
+        import librosa
+        import soundfile as sf
         import requests as _requests
+        import torch
 
         audio_bytes = None
 
@@ -57,15 +88,23 @@ class CohereTranscriber:
                 status_code=400,
             )
 
-        audio = load_audio(io.BytesIO(audio_bytes), sampling_rate=SAMPLE_RATE)
+        audio, sr = sf.read(io.BytesIO(audio_bytes))
+        if sr != SAMPLE_RATE:
+            audio = librosa.resample(audio, orig_sr=sr, target_sr=SAMPLE_RATE)
+        if audio.ndim > 1:
+            audio = audio.mean(axis=1)
 
         inputs = self.processor(
-            audio, sampling_rate=SAMPLE_RATE, return_tensors="pt", language="en"
+            audio, sampling_rate=SAMPLE_RATE, return_tensors="pt"
         )
-        inputs.to(self.model.device, dtype=self.model.dtype)
+        input_features = inputs.input_features.to(self.device, dtype=torch.bfloat16)
+
+        with torch.no_grad():
+            generated_ids = self.model.generate(input_features)
 
-        outputs = self.model.generate(**inputs, max_new_tokens=256)
-        transcription = self.processor.decode(outputs, skip_special_tokens=True)
+        transcription = self.processor.batch_decode(
+            generated_ids, skip_special_tokens=True
+        )[0]
 
         return JSONResponse({"transcription": transcription})
 

vllm_inference.py

@@ -27,7 +27,7 @@ FAST_BOOT = False
 SPECULATIVE_MODEL_NAME = "google/gemma-4-26B-A4B-it-assistant"
 SPECULATIVE_MODEL_REVISION = "f188f476dc11dd5bb3014dc861529d316bce49d3"
 
-app = modal.App("vllm-inference")
+app = modal.App("example-vllm-inference")
 
 N_GPU = 1
 MINUTES = 60