Spaces:
Sleeping
Sleeping
File size: 3,538 Bytes
732b14f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 | """Throttled OpenAI chat completions for auxiliary LLM call sites."""
from __future__ import annotations
from typing import Any
from app.async_executor import run_sync_in_executor
from app.config import settings
from app.llm.llm_throttle import throttled_llm_call
async def chat_completions_create_raw(
*,
messages: list[Any],
model: str | None = None,
max_tokens: int = 800,
temperature: float = 0.0,
response_format: dict[str, str] | None = None,
phase: str = "chat",
section_id: str | None = None,
api_key: str | None = None,
tenant_id: str | None = None,
) -> str:
"""Chat completion with arbitrary message content (text or multimodal)."""
model_name = model or settings.chat_model
key = (api_key or settings.openai_api_key or "").strip()
if not key:
return ""
kwargs: dict[str, Any] = {
"model": model_name,
"messages": messages,
"max_tokens": max_tokens,
"temperature": temperature,
}
if response_format is not None:
kwargs["response_format"] = response_format
if settings.enable_async_pipeline:
from openai import AsyncOpenAI
from app.llm.llm_throttle import make_cache_hit_slot
from app.llm.prompt_cache import (
log_openai_cache_usage,
normalize_messages_for_caching,
openai_extra_kwargs,
prompt_caching_active,
)
client = AsyncOpenAI(api_key=key)
if prompt_caching_active() and isinstance(messages, list):
kwargs["messages"] = normalize_messages_for_caching(messages)
kwargs.update(
openai_extra_kwargs(
phase=phase, model=model_name, tenant_id=tenant_id
)
)
cache_slot = make_cache_hit_slot()
async def _call() -> str:
resp = await client.chat.completions.create(**kwargs)
if prompt_caching_active():
cache_slot[0] = log_openai_cache_usage(
resp, phase=phase, section_id=section_id
)
return (resp.choices[0].message.content or "").strip()
return await throttled_llm_call(
phase=phase,
section_id=section_id,
cache_hit_out=cache_slot,
call=_call,
)
def _sync() -> str:
from openai import OpenAI
from app.llm.llm_throttle import throttled_sync_llm_call
client = OpenAI(api_key=key)
def _call() -> str:
resp = client.chat.completions.create(**kwargs)
return (resp.choices[0].message.content or "").strip()
return throttled_sync_llm_call(phase=phase, section_id=section_id, call=_call)
return await run_sync_in_executor(_sync)
async def chat_completions_create(
*,
messages: list[dict[str, str]],
model: str | None = None,
max_tokens: int = 800,
temperature: float = 0.0,
response_format: dict[str, str] | None = None,
phase: str = "chat",
section_id: str | None = None,
api_key: str | None = None,
tenant_id: str | None = None,
) -> str:
"""Return assistant message content from a text-only chat completion."""
return await chat_completions_create_raw(
messages=messages,
model=model,
max_tokens=max_tokens,
temperature=temperature,
response_format=response_format,
api_key=api_key,
phase=phase,
section_id=section_id,
tenant_id=tenant_id,
)
|