Revert to simpler configuration - optimizations caused slowdown

- Reverted n_ctx back to 4096 (2048 was too small)
- Reverted streaming to simple token-by-token (buffering caused issues)
- Reduced threads to 6 (8 may have been overloading)
- Kept max_tokens at 256 (only change that should help)

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

llm_handler.py

@@ -51,9 +51,9 @@ class CybersecurityLLM:
         logger.info("Initializing model...")
         self.llm = Llama(
             model_path=model_path,
-            n_ctx=2048,  # Reduced context window for faster prompt processing
+            n_ctx=4096,  # Context window
             n_batch=512,  # Batch size for prompt processing
-            n_threads=8,  # Use all 8 vCPUs for maximum inference speed
+            n_threads=6,  # Use 6 of 8 vCPUs (leave 2 for system)
             n_gpu_layers=0,  # CPU only
             seed=-1,  # Random seed
             f16_kv=True,  # Use f16 for key/value cache (saves memory)

main.py

@@ -328,29 +328,18 @@ async def chat_stream(request: ChatRequest):
     async def generate():
         try:
             full_response = ""
-            buffer = ""
-            buffer_size = 3  # Send every 3 tokens for better perceived speed
 
             # Send initial metadata
             yield f"data: {json.dumps({'type': 'start', 'session_id': session_id, 'model': MODEL_REPO, 'interaction_count': count})}\n\n"
 
-            # Stream tokens with buffering
+            # Stream tokens
             for token in llm_instance.generate_stream(
                     request.message,
                     max_tokens=request.max_tokens
             ):
                 full_response += token
-                buffer += token
-
-                # Send buffer when it reaches buffer_size or contains whitespace
-                if len(buffer) >= buffer_size or ' ' in token or '\n' in token:
-                    yield f"data: {json.dumps({'type': 'token', 'content': buffer})}\n\n"
-                    buffer = ""
-                    await asyncio.sleep(0)
-
-            # Send any remaining buffered tokens
-            if buffer:
-                yield f"data: {json.dumps({'type': 'token', 'content': buffer})}\n\n"
+                yield f"data: {json.dumps({'type': 'token', 'content': token})}\n\n"
+                await asyncio.sleep(0)
 
             # Log interaction
             log_interaction(session_id, request.message, len(full_response))