JatsTheAIGen commited on
Commit
42ed1a7
·
1 Parent(s): 927854c

Fix max_tokens calculation to respect model context window

Browse files
Files changed (1) hide show
  1. src/llm_router.py +7 -7
src/llm_router.py CHANGED
@@ -54,7 +54,7 @@ class LLMRouter:
54
  logger.info(f"✓ Novita AI API client initialized")
55
  logger.info(f" Base URL: {self.settings.novita_base_url}")
56
  logger.info(f" Model: {self.settings.novita_model}")
57
- except Exception as e:
58
  logger.error(f"Failed to initialize Novita AI client: {e}")
59
  raise RuntimeError(f"Could not initialize Novita AI API client: {e}") from e
60
 
@@ -162,8 +162,8 @@ class LLMRouter:
162
 
163
  except Exception as e:
164
  logger.error(f"Error calling Novita AI API: {e}", exc_info=True)
165
- raise
166
-
167
  def _format_deepseek_r1_prompt(self, prompt: str, task_type: str, model_config: dict) -> str:
168
  """
169
  Format prompt according to DeepSeek-R1 best practices:
@@ -318,7 +318,7 @@ class LLMRouter:
318
  continue
319
 
320
  # Estimate tokens (simple: 1 token ≈ 4 chars)
321
- tokens = len(content) // 4
322
 
323
  if total_tokens + tokens <= max_tokens:
324
  formatted_context.append(f"=== {element.upper()} ===\n{content}")
@@ -337,7 +337,7 @@ class LLMRouter:
337
  def _truncate_to_tokens(self, content: str, max_tokens: int) -> str:
338
  """Truncate content to fit within token limit"""
339
  # Simple character-based truncation (1 token ≈ 4 chars)
340
- max_chars = max_tokens * 4
341
- if len(content) <= max_chars:
342
- return content
343
  return content[:max_chars - 3] + "..."
 
54
  logger.info(f"✓ Novita AI API client initialized")
55
  logger.info(f" Base URL: {self.settings.novita_base_url}")
56
  logger.info(f" Model: {self.settings.novita_model}")
57
+ except Exception as e:
58
  logger.error(f"Failed to initialize Novita AI client: {e}")
59
  raise RuntimeError(f"Could not initialize Novita AI API client: {e}") from e
60
 
 
162
 
163
  except Exception as e:
164
  logger.error(f"Error calling Novita AI API: {e}", exc_info=True)
165
+ raise
166
+
167
  def _format_deepseek_r1_prompt(self, prompt: str, task_type: str, model_config: dict) -> str:
168
  """
169
  Format prompt according to DeepSeek-R1 best practices:
 
318
  continue
319
 
320
  # Estimate tokens (simple: 1 token ≈ 4 chars)
321
+ tokens = len(content) // 4
322
 
323
  if total_tokens + tokens <= max_tokens:
324
  formatted_context.append(f"=== {element.upper()} ===\n{content}")
 
337
  def _truncate_to_tokens(self, content: str, max_tokens: int) -> str:
338
  """Truncate content to fit within token limit"""
339
  # Simple character-based truncation (1 token ≈ 4 chars)
340
+ max_chars = max_tokens * 4
341
+ if len(content) <= max_chars:
342
+ return content
343
  return content[:max_chars - 3] + "..."