truncates

app.py

@@ -29,7 +29,8 @@ tokenizer = AutoTokenizer.from_pretrained(model_name)
 
 # Token constraints
 MAX_MODEL_TOKENS = 1024
-SAFE_CHUNK_SIZE = 600  # Reduced to leave room for special tokens
+SAFE_CHUNK_SIZE = 600  # Safe for aggregation
+TRUNCATED_TOKENS = MAX_MODEL_TOKENS - 2  # Leave room for special tokens
 
 # Pydantic schemas
 class SummarizationItem(BaseModel):
@@ -79,6 +80,14 @@ def split_sentences(text: str, max_sentence_tokens: int = SAFE_CHUNK_SIZE) -> li
 
     return split_results
 
+# Truncate text safely at token-level
+def truncate_text(text: str, max_tokens: int = TRUNCATED_TOKENS) -> str:
+    tokens = tokenizer.encode(text, add_special_tokens=False)
+    if len(tokens) <= max_tokens:
+        return text
+    truncated = tokens[:max_tokens]
+    return tokenizer.decode(truncated, skip_special_tokens=True)
+
 # Chunking based on token length
 def chunk_text(text: str, max_tokens: int = SAFE_CHUNK_SIZE) -> List[str]:
     sentences = split_sentences(text)
@@ -121,7 +130,7 @@ async def summarize_batch(request: BatchSummarizationRequest):
         logger.info(f"[CHUNKING] content_id={item.content_id} num_chunks={len(chunks)}")
 
         for chunk in chunks:
-            all_chunks.append(chunk)
+            all_chunks.append(truncate_text(chunk))  # ✅ enforce max length
             chunk_map.append(item.content_id)
 
     if not all_chunks: