"""
Shared vLLM client and tier-aware model calling utilities.
All LLM inference across OncoAgent flows through this module,
ensuring consistent model selection, error handling, and
environment variable management.
Design inspired by:
- Hermes Agent: structured tool calling with JSON output
- Claude Code: deterministic harness separating LLM from execution
Production target: AMD Instinct MI300X via ROCm 7.2 + vLLM
Development fallback: Featherless.ai OpenAI-compatible API
"""
import os
import re
import logging
from typing import Optional, Dict, Any, List
from dataclasses import dataclass
from openai import OpenAI
from dotenv import load_dotenv
load_dotenv()
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Tier Configuration
# ---------------------------------------------------------------------------
@dataclass(frozen=True)
class TierSpec:
"""Immutable specification for a model tier."""
tier_id: int
name: str
model_id: str
description: str
max_tokens: int
temperature: float
def __str__(self) -> str:
return f"Tier {self.tier_id}: {self.name} ({self.model_id})"
# Production tier definitions — Qwen 3.5 / 3.6 as per project rules
TIER_SPECS: Dict[int, TierSpec] = {
1: TierSpec(
tier_id=1,
name="Speed Triage",
model_id=os.getenv("TIER1_MODEL_ID", "Qwen/Qwen3.5-9B"),
description="Fast model for initial triage and low-complexity cases.",
max_tokens=2048,
temperature=0.1,
),
2: TierSpec(
tier_id=2,
name="Deep Reasoning",
model_id=os.getenv("TIER2_MODEL_ID", "Qwen/Qwen3.6-27B"),
description="High-reasoning model for complex oncology cases and validation.",
max_tokens=4096,
temperature=0.0,
),
}
# ---------------------------------------------------------------------------
# Qwen3 Thinking-Mode Handler
# ---------------------------------------------------------------------------
_THINK_PATTERN = re.compile(r".*?", re.DOTALL)
def _strip_thinking_tokens(text: str) -> str:
"""Remove Qwen3 ... blocks from model output.
Qwen3 models use an internal reasoning mode that wraps chain-of-thought
in tags. We preserve only the final answer for the pipeline.
Args:
text: Raw model output potentially containing blocks.
Returns:
Cleaned text with thinking blocks removed.
"""
cleaned = _THINK_PATTERN.sub("", text).strip()
# If everything was inside tags, return the original
return cleaned if cleaned else text.strip()
# ---------------------------------------------------------------------------
# vLLM Client Singleton
# ---------------------------------------------------------------------------
_vllm_client: Optional[OpenAI] = None
def get_vllm_client() -> OpenAI:
"""Return a cached OpenAI-compatible client pointing at vLLM.
Reads ``VLLM_API_BASE`` and ``VLLM_API_KEY`` from environment.
Returns:
OpenAI client configured for the local vLLM server.
"""
global _vllm_client
if _vllm_client is None:
api_base = os.getenv("VLLM_API_BASE", "http://localhost:8000/v1")
api_key = os.getenv("VLLM_API_KEY", "EMPTY")
_vllm_client = OpenAI(base_url=api_base, api_key=api_key)
logger.info("vLLM client initialised → %s", api_base)
return _vllm_client
# ---------------------------------------------------------------------------
# Model ID Resolution (handles Featherless fallback for dev)
# ---------------------------------------------------------------------------
def _resolve_model_id(spec: TierSpec) -> str:
"""Resolve the actual model ID to use for API calls.
In production (local vLLM), we use the exact model ID.
In development (Featherless.ai), some models may not be available,
so we check for configured fallbacks.
Args:
spec: The TierSpec for the requested tier.
Returns:
The model ID string to pass to the API.
"""
api_base = os.getenv("VLLM_API_BASE", "http://localhost:8000/v1")
is_featherless = "featherless" in api_base.lower()
if is_featherless and spec.tier_id == 2:
# Qwen3.6-27B is not available on Featherless — use fallback
fallback = os.getenv("TIER2_FEATHERLESS_FALLBACK", "Qwen/Qwen3.5-27B")
logger.info(
"Featherless detected: Tier 2 fallback %s → %s",
spec.model_id, fallback,
)
return fallback
return spec.model_id
# ---------------------------------------------------------------------------
# Local Adapter Manager (PEFT — AMD MI300X only)
# ---------------------------------------------------------------------------
class LocalModelManager:
"""Singleton to manage local LoRA model loading and inference.
Only used on the AMD droplet with working ROCm/GPU drivers.
In development without GPU, this is skipped entirely.
"""
_instance = None
def __new__(cls):
if cls._instance is None:
cls._instance = super(LocalModelManager, cls).__new__(cls)
cls._instance.model = None
cls._instance.tokenizer = None
cls._instance.initialized = False
return cls._instance
def initialize(self) -> None:
"""Load the base model and LoRA adapters."""
if self.initialized:
return
try:
from transformers import AutoModelForCausalLM, AutoTokenizer
from peft import PeftModel
import torch
except ImportError:
logger.warning("Transformers/PEFT/Torch not installed. Local inference disabled.")
return
adapter_path = os.getenv("LOCAL_ADAPTER_PATH")
base_model_id = os.getenv("BASE_MODEL_ID", "Qwen/Qwen3.5-9B")
if not adapter_path or not os.path.exists(adapter_path):
logger.error("Local adapter path not found: %s", adapter_path)
return
logger.info("Loading base model %s + adapters %s...", base_model_id, adapter_path)
try:
self.tokenizer = AutoTokenizer.from_pretrained(
base_model_id, trust_remote_code=True,
)
base_model = AutoModelForCausalLM.from_pretrained(
base_model_id,
dtype=torch.bfloat16,
device_map="auto",
trust_remote_code=True,
)
self.model = PeftModel.from_pretrained(base_model, adapter_path)
self.model.eval()
self.initialized = True
logger.info("Local BF16 model ready on %s", os.getenv("DEVICE", "cuda"))
except Exception as exc:
logger.error("Failed to load local model: %s", exc)
def generate(
self,
system_prompt: str,
user_prompt: str,
max_tokens: int,
temperature: float,
) -> str:
"""Run inference using the loaded local model."""
if not self.initialized:
self.initialize()
if not self.initialized:
raise RuntimeError("Local model manager not initialized.")
import torch
messages = [
{"role": "system", "content": system_prompt},
{"role": "user", "content": user_prompt},
]
prompt_str = self.tokenizer.apply_chat_template(
messages, tokenize=False, add_generation_prompt=True,
)
inputs = self.tokenizer(text=prompt_str, return_tensors="pt").to("cuda")
with torch.no_grad():
outputs = self.model.generate(
**inputs,
max_new_tokens=max_tokens,
temperature=temperature,
do_sample=temperature > 0,
use_cache=True,
pad_token_id=self.tokenizer.pad_token_id,
)
generated_ids = outputs[:, inputs.input_ids.shape[1]:]
response = self.tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
return _strip_thinking_tokens(response)
_local_manager = LocalModelManager()
# ---------------------------------------------------------------------------
# Tier-Aware Model Calling
# ---------------------------------------------------------------------------
def call_tier_model(
tier: int,
system_prompt: str,
user_prompt: str,
max_tokens: Optional[int] = None,
temperature: Optional[float] = None,
json_mode: bool = False,
) -> str:
"""Call the appropriate model based on the selected tier.
This is the *single entry point* for all LLM inference in OncoAgent.
Every node must call this function instead of instantiating clients.
Flow:
1. If USE_LOCAL_ADAPTERS=true AND tier=1 → try local PEFT inference
2. If local fails or not enabled → route through vLLM/Featherless API
Args:
tier: Model tier (1 = fast 9B, 2 = deep 27B).
system_prompt: System-level instructions.
user_prompt: User-level content / query.
max_tokens: Override the tier's default max_tokens.
temperature: Override the tier's default temperature.
json_mode: If True, request JSON response format.
Returns:
The model's text response (stripped of thinking tokens).
Raises:
ValueError: If the tier is not 1 or 2.
RuntimeError: If the vLLM server is unreachable.
"""
spec = TIER_SPECS.get(tier)
if spec is None:
raise ValueError(f"Invalid tier {tier}. Must be 1 or 2.")
effective_max_tokens = max_tokens or spec.max_tokens
effective_temperature = temperature if temperature is not None else spec.temperature
logger.info(
"Calling %s (max_tokens=%d, temp=%.2f, json=%s)",
spec, effective_max_tokens, effective_temperature, json_mode,
)
# --- Path 1: Local LoRA adapters (MI300X only) ---
use_local = os.getenv("USE_LOCAL_ADAPTERS", "false").lower() == "true"
if tier == 1 and use_local:
try:
logger.info("Routing Tier 1 to local LoRA adapters...")
return _local_manager.generate(
system_prompt=system_prompt,
user_prompt=user_prompt,
max_tokens=effective_max_tokens,
temperature=effective_temperature,
)
except Exception as local_exc:
logger.warning("Local inference failed, falling back to API: %s", local_exc)
# --- Path 2: vLLM / Featherless API ---
model_id = _resolve_model_id(spec)
try:
client = get_vllm_client()
kwargs: Dict[str, Any] = {
"model": model_id,
"messages": [
{"role": "system", "content": system_prompt},
{"role": "user", "content": user_prompt},
],
"temperature": effective_temperature,
"max_tokens": effective_max_tokens,
}
if json_mode:
kwargs["response_format"] = {"type": "json_object"}
response = client.chat.completions.create(**kwargs)
raw_text = response.choices[0].message.content or ""
text = _strip_thinking_tokens(raw_text)
if not text:
logger.warning(
"Model returned empty response (raw_len=%d). "
"May be all tokens. Returning raw.",
len(raw_text),
)
text = raw_text.strip() if raw_text else ""
logger.debug("Response length: %d chars", len(text))
return text
except Exception as exc:
logger.error("vLLM call failed for %s: %s", spec, exc)
raise RuntimeError(
f"Error connecting to vLLM ({model_id}): {exc}"
) from exc
def get_tier_spec(tier: int) -> TierSpec:
"""Retrieve the TierSpec for the given tier number.
Args:
tier: 1 or 2.
Returns:
The corresponding TierSpec.
"""
spec = TIER_SPECS.get(tier)
if spec is None:
raise ValueError(f"Invalid tier {tier}. Available: {list(TIER_SPECS.keys())}")
return spec