Commit ·
42ed1a7
1
Parent(s): 927854c
Fix max_tokens calculation to respect model context window
Browse files- src/llm_router.py +7 -7
src/llm_router.py
CHANGED
|
@@ -54,7 +54,7 @@ class LLMRouter:
|
|
| 54 |
logger.info(f"✓ Novita AI API client initialized")
|
| 55 |
logger.info(f" Base URL: {self.settings.novita_base_url}")
|
| 56 |
logger.info(f" Model: {self.settings.novita_model}")
|
| 57 |
-
|
| 58 |
logger.error(f"Failed to initialize Novita AI client: {e}")
|
| 59 |
raise RuntimeError(f"Could not initialize Novita AI API client: {e}") from e
|
| 60 |
|
|
@@ -162,8 +162,8 @@ class LLMRouter:
|
|
| 162 |
|
| 163 |
except Exception as e:
|
| 164 |
logger.error(f"Error calling Novita AI API: {e}", exc_info=True)
|
| 165 |
-
|
| 166 |
-
|
| 167 |
def _format_deepseek_r1_prompt(self, prompt: str, task_type: str, model_config: dict) -> str:
|
| 168 |
"""
|
| 169 |
Format prompt according to DeepSeek-R1 best practices:
|
|
@@ -318,7 +318,7 @@ class LLMRouter:
|
|
| 318 |
continue
|
| 319 |
|
| 320 |
# Estimate tokens (simple: 1 token ≈ 4 chars)
|
| 321 |
-
|
| 322 |
|
| 323 |
if total_tokens + tokens <= max_tokens:
|
| 324 |
formatted_context.append(f"=== {element.upper()} ===\n{content}")
|
|
@@ -337,7 +337,7 @@ class LLMRouter:
|
|
| 337 |
def _truncate_to_tokens(self, content: str, max_tokens: int) -> str:
|
| 338 |
"""Truncate content to fit within token limit"""
|
| 339 |
# Simple character-based truncation (1 token ≈ 4 chars)
|
| 340 |
-
|
| 341 |
-
|
| 342 |
-
|
| 343 |
return content[:max_chars - 3] + "..."
|
|
|
|
| 54 |
logger.info(f"✓ Novita AI API client initialized")
|
| 55 |
logger.info(f" Base URL: {self.settings.novita_base_url}")
|
| 56 |
logger.info(f" Model: {self.settings.novita_model}")
|
| 57 |
+
except Exception as e:
|
| 58 |
logger.error(f"Failed to initialize Novita AI client: {e}")
|
| 59 |
raise RuntimeError(f"Could not initialize Novita AI API client: {e}") from e
|
| 60 |
|
|
|
|
| 162 |
|
| 163 |
except Exception as e:
|
| 164 |
logger.error(f"Error calling Novita AI API: {e}", exc_info=True)
|
| 165 |
+
raise
|
| 166 |
+
|
| 167 |
def _format_deepseek_r1_prompt(self, prompt: str, task_type: str, model_config: dict) -> str:
|
| 168 |
"""
|
| 169 |
Format prompt according to DeepSeek-R1 best practices:
|
|
|
|
| 318 |
continue
|
| 319 |
|
| 320 |
# Estimate tokens (simple: 1 token ≈ 4 chars)
|
| 321 |
+
tokens = len(content) // 4
|
| 322 |
|
| 323 |
if total_tokens + tokens <= max_tokens:
|
| 324 |
formatted_context.append(f"=== {element.upper()} ===\n{content}")
|
|
|
|
| 337 |
def _truncate_to_tokens(self, content: str, max_tokens: int) -> str:
|
| 338 |
"""Truncate content to fit within token limit"""
|
| 339 |
# Simple character-based truncation (1 token ≈ 4 chars)
|
| 340 |
+
max_chars = max_tokens * 4
|
| 341 |
+
if len(content) <= max_chars:
|
| 342 |
+
return content
|
| 343 |
return content[:max_chars - 3] + "..."
|