Spaces:
Sleeping
Sleeping
File size: 4,135 Bytes
578da72 be70bd4 578da72 be70bd4 578da72 be70bd4 578da72 923dda1 578da72 923dda1 578da72 923dda1 578da72 68ee171 923dda1 578da72 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 | from fastapi import APIRouter, Body, HTTPException
from app.config import get_settings
from app.llm.improved_gemini_client import ImprovedGeminiClient
from app.llm.improved_ollama_client import ImprovedOllamaClient
from app.llm.improved_vllm_client import ImprovedVllmClient
from app.models.default_personas import get_default_personas
from app.core.bootstrap import chat_orchestrator, llm, current_provider, available_providers
from pydantic import BaseModel
import os
import logging
logger = logging.getLogger(__name__)
router = APIRouter()
def create_llm_client(provider: str = None):
global current_provider
if provider is None:
provider = current_provider
if provider == "gemini":
try:
return ImprovedGeminiClient(model_name=os.getenv("GEMINI_MODEL"))
except ValueError as e:
logger.warning(f"Gemini API key not found, falling back to Ollama: {e}")
return ImprovedOllamaClient(model_name="llama3.2:1b")
elif provider == "ollama":
return ImprovedOllamaClient(model_name="llama3.2:1b")
elif provider == "vllm":
settings = get_settings()
if not settings.llm.vllm.api_url:
raise ValueError("No vLLM endpoint configured. Set llm.vllm.api_url in your config.")
return ImprovedVllmClient(
api_url=settings.llm.vllm.api_url,
api_key=settings.llm.vllm.api_key,
)
else:
raise ValueError(f"Unknown provider: {provider}")
# NOTE: Personas and `llm` are already created and registered by
# app/core/bootstrap.py with the correct ResilientLLMClient wrapping
# (primary vLLM + OpenAI fallback). Re-registering them here at module
# import time replaces them with a raw vLLM client that lacks the
# fallback wrapper AND lacks the api_username/api_key wiring, breaking
# both auth and failover. Bootstrap already handles initial setup.
class ProviderSwitch(BaseModel):
provider: str
@router.get("/current-provider")
async def get_current_provider():
return {
"current_provider": current_provider,
"available_providers": available_providers,
"model_info": {
"name": llm.model_name if hasattr(llm, 'model_name') else "gemini-2.0-flash",
"provider": current_provider
}
}
@router.post("/switch-provider")
async def switch_provider(provider_data: ProviderSwitch):
global current_provider, llm
if provider_data.provider not in available_providers:
raise HTTPException(status_code=400, detail=f"Unknown provider: {provider_data.provider}. Available: {available_providers}")
try:
from app.core.bootstrap import (
create_orchestrator_llm,
create_persona_llm,
)
current_provider = provider_data.provider
new_llm = create_orchestrator_llm()
llm = new_llm
chat_orchestrator.llm_client = new_llm
persona_llm = create_persona_llm()
new_personas = get_default_personas(persona_llm)
chat_orchestrator.personas.clear()
for persona in new_personas:
chat_orchestrator.register_persona(persona)
return {
"message": f"Successfully switched to {current_provider}",
"current_provider": current_provider,
"model_info": {
"name": new_llm.model_name if hasattr(new_llm, 'model_name') else "gemini-2.0-flash",
"provider": current_provider
}
}
except Exception as e:
raise HTTPException(status_code=500, detail=f"Failed to switch to {provider_data.provider}: {str(e)}")
@router.post("/switch-model")
async def switch_model(model_name: str = Body(...)):
if "gemini" in model_name.lower():
return await switch_provider(ProviderSwitch(provider="gemini"))
else:
return await switch_provider(ProviderSwitch(provider="ollama"))
@router.get("/current-model")
async def get_current_model():
model_name = llm.model_name if hasattr(llm, 'model_name') else "gemini-2.0-flash"
return {
"model": model_name,
"provider": current_provider
}
|