import asyncio import requests from tenacity import ( retry, stop_after_attempt, wait_random_exponential, ) from openai import OpenAI, Stream from openai.types.chat import ChatCompletion from typing import Optional, List from litellm import token_counter from ..core.registry import register_model from .model_configs import OpenRouterConfig from .base_model import BaseLLM from .model_utils import Cost, cost_manager @register_model(config_cls=OpenRouterConfig, alias=["openrouter"]) class OpenRouterLLM(BaseLLM): def init_model(self): config: OpenRouterConfig = self.config self._client = self._init_client(config) self._default_ignore_fields = ["llm_type", "openrouter_key", "openrouter_base", "openrouter_model_base", "output_response"] def _init_client(self, config: OpenRouterConfig): client = OpenAI( api_key=config.openrouter_key, base_url=config.openrouter_base ) return client def formulate_messages(self, prompts: List[str], system_messages: Optional[List[str]] = None) -> List[List[dict]]: if system_messages: assert len(prompts) == len(system_messages), f"the number of prompts ({len(prompts)}) is different from the number of system_messages ({len(system_messages)})" else: system_messages = [None] * len(prompts) messages_list = [] for prompt, system_message in zip(prompts, system_messages): messages = [] if system_message: messages.append({"role": "system", "content": system_message}) messages.append({"role": "user", "content": prompt}) messages_list.append(messages) return messages_list def update_completion_params(self, params1: dict, params2: dict) -> dict: config_params: list = self.config.get_config_params() for key, value in params2.items(): if key in self._default_ignore_fields: continue if key not in config_params: continue params1[key] = value return params1 def get_completion_params(self, **kwargs): completion_params = self.config.get_set_params(ignore=self._default_ignore_fields) completion_params = self.update_completion_params(completion_params, kwargs) return completion_params def get_stream_output(self, response: Stream, output_response: bool=True) -> str: output = "" for chunk in response: content = chunk.choices[0].delta.content if content: if output_response: print(content, end="", flush=True) output += content if output_response: print("") return output async def get_stream_output_async(self, response, output_response: bool = False) -> str: output = "" async for chunk in response: content = chunk.choices[0].delta.content if content: if output_response: print(content, end="", flush=True) output += content if output_response: print("") return output def get_completion_output(self, response: ChatCompletion, output_response: bool=True) -> str: output = response.choices[0].message.content if output_response: print(output) return output @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(5)) def single_generate(self, messages: List[dict], **kwargs) -> str: stream = kwargs.get("stream", self.config.stream) output_response = kwargs.get("output_response", self.config.output_response) try: completion_params = self.get_completion_params(**kwargs) response = self._client.chat.completions.create(messages=messages, **completion_params) if stream: output = self.get_stream_output(response, output_response=output_response) cost = self._stream_cost(messages=messages, output=output) else: output: str = self.get_completion_output(response=response, output_response=output_response) cost = self._completion_cost(response) self._update_cost(cost=cost) except Exception as e: raise RuntimeError(f"Error during single_generate of OpenRouterLLM: {str(e)}") return output def batch_generate(self, batch_messages: List[List[dict]], **kwargs) -> List[str]: return [self.single_generate(messages=one_messages, **kwargs) for one_messages in batch_messages] async def single_generate_async(self, messages: List[dict], **kwargs) -> str: stream = kwargs.get("stream", self.config.stream) output_response = kwargs.get("output_response", self.config.output_response) try: isolated_client = self._init_client(self.config) completion_params = self.get_completion_params(**kwargs) loop = asyncio.get_event_loop() response = await loop.run_in_executor( None, lambda: isolated_client.chat.completions.create( messages=messages, **completion_params ) ) if stream: if hasattr(response, "__aiter__"): output = await self.get_stream_output_async(response, output_response=output_response) else: output = self.get_stream_output(response, output_response=output_response) cost = self._stream_cost(messages=messages, output=output) else: output: str = self.get_completion_output(response=response, output_response=output_response) cost = self._completion_cost(response) self._update_cost(cost=cost) except Exception as e: raise RuntimeError(f"Error during single_generate_async of OpenRouterLLM: {str(e)}") return output def _completion_cost(self, response: ChatCompletion) -> Cost: input_tokens = response.usage.prompt_tokens output_tokens = response.usage.completion_tokens return self._compute_cost(input_tokens=input_tokens, output_tokens=output_tokens) def _stream_cost(self, messages: List[dict], output: str) -> Cost: model: str = self.config.model input_tokens = token_counter(model=model, messages=messages) output_tokens = token_counter(model=model, text=output) return self._compute_cost(input_tokens=input_tokens, output_tokens=output_tokens) def _compute_cost(self, input_tokens: int, output_tokens: int) -> Cost: input_cost_per_token, output_cost_per_token = self._get_cost() input_cost = input_tokens * input_cost_per_token output_cost = output_tokens * output_cost_per_token cost = Cost(input_tokens=input_tokens, output_tokens=output_tokens, input_cost=input_cost, output_cost=output_cost) return cost def _update_cost(self, cost: Cost): cost_manager.update_cost(cost=cost, model=self.config.model) def _get_cost(self): url = self.config.openrouter_model_base response = requests.get(url) data = response.json() for model in data['data']: if model['id'] == self.config.model: pricing = model.get('pricing',{}) input_cost = float(pricing.get('prompt', 0)) output_cost = float(pricing.get('completion', 0)) return input_cost, output_cost return 0, 0