Cyber Catalyst Team commited on
Commit
111ac7a
·
1 Parent(s): 787cf39

feat: route Mistral models natively to Mistral API and map model names correctly

Browse files
Files changed (1) hide show
  1. backend.py +11 -3
backend.py CHANGED
@@ -715,11 +715,19 @@ async def chat_completions(request: Request, authorization: str = Header(None)):
715
  stream = body.get("stream", False)
716
  session_id = body.get("session_id") or str(uuid.uuid4())
717
 
 
 
 
 
 
 
 
 
718
  is_agentic = requested_model in TOOL_CAPABLE_MODELS
719
  request_id = str(uuid.uuid4())[:8]
720
 
721
  ACTIVE_SESSIONS.add(session_id)
722
- log_activity(f"Session [{session_id[:6]}] connected. Model: {requested_model}")
723
 
724
  # Build message history
725
  final_messages = []
@@ -760,7 +768,7 @@ async def chat_completions(request: Request, authorization: str = Header(None)):
760
  kwargs["tools"] = TOOLS
761
  kwargs["tool_choice"] = "auto"
762
  async with completions_semaphore:
763
- response = await nim_client.chat.completions.create(**kwargs)
764
  content = response.choices[0].message.content or ""
765
  await save_message(session_id, "assistant", content)
766
  ACTIVE_SESSIONS.discard(session_id)
@@ -793,7 +801,7 @@ async def chat_completions(request: Request, authorization: str = Header(None)):
793
  full_content = ""
794
  tool_calls_raw = {} # index -> {id, name, arguments_str}
795
 
796
- async for chunk in await nim_client.chat.completions.create(**kwargs):
797
  choice = chunk.choices[0] if chunk.choices else None
798
  if not choice:
799
  continue
 
715
  stream = body.get("stream", False)
716
  session_id = body.get("session_id") or str(uuid.uuid4())
717
 
718
+ # Route Mistral queries natively to the Mistral API
719
+ client = nim_client
720
+ if "mistral" in requested_model.lower():
721
+ if mistral_client:
722
+ client = mistral_client
723
+ if requested_model == "mistralai/mistral-large-2-instruct":
724
+ requested_model = "mistral-large-latest"
725
+
726
  is_agentic = requested_model in TOOL_CAPABLE_MODELS
727
  request_id = str(uuid.uuid4())[:8]
728
 
729
  ACTIVE_SESSIONS.add(session_id)
730
+ log_activity(f"Session [{session_id[:6]}] connected. Model: {requested_model} | Provider: {'Mistral' if client == mistral_client else 'NIM'}")
731
 
732
  # Build message history
733
  final_messages = []
 
768
  kwargs["tools"] = TOOLS
769
  kwargs["tool_choice"] = "auto"
770
  async with completions_semaphore:
771
+ response = await client.chat.completions.create(**kwargs)
772
  content = response.choices[0].message.content or ""
773
  await save_message(session_id, "assistant", content)
774
  ACTIVE_SESSIONS.discard(session_id)
 
801
  full_content = ""
802
  tool_calls_raw = {} # index -> {id, name, arguments_str}
803
 
804
+ async for chunk in await client.chat.completions.create(**kwargs):
805
  choice = chunk.choices[0] if chunk.choices else None
806
  if not choice:
807
  continue