Optimize model parameters for faster CPU inference

- Reduced n_ctx from 4096 to 2048 for faster inference
- Reduced n_batch from 512 to 256 for lower memory usage
- Reduced n_threads from 8 to 4 for HF Spaces CPU limits
- Changed f16_kv to False for better CPU performance
- Reduced default max_tokens from 512 to 256 for faster responses

These changes need to be rebuilt into the base Docker image.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

llm_handler.py

@@ -51,12 +51,12 @@ class CybersecurityLLM:
         logger.info("Initializing model...")
         self.llm = Llama(
             model_path=model_path,
-            n_ctx=4096,  # Context window
-            n_batch=512,  # Batch size for prompt processing
-            n_threads=8,  # Adjust based on CPU cores
+            n_ctx=2048,  # Reduced context window for faster inference
+            n_batch=256,  # Smaller batch size for lower memory usage
+            n_threads=4,  # Reduced threads for HF Spaces CPU limits
             n_gpu_layers=0,  # CPU only
             seed=-1,  # Random seed
-            f16_kv=True,  # Use f16 for key/value cache
+            f16_kv=False,  # Use f32 for better CPU performance
             logits_all=False,  # Only compute logits for last token
             vocab_only=False,  # Load full model
             use_mmap=True,  # Memory-map model for efficiency

main.py

@@ -158,7 +158,7 @@ app.add_middleware(
 class ChatRequest(BaseModel):
     message: str = Field(..., description="User's security question")
     session_id: Optional[str] = Field(None, description="Session ID for conversation continuity")
-    max_tokens: Optional[int] = Field(512, description="Maximum response length")
+    max_tokens: Optional[int] = Field(256, description="Maximum response length")
     temperature: Optional[float] = Field(0.7, description="Response creativity (0-1)")
     use_rag: Optional[bool] = Field(True, description="Use RAG for enhanced accuracy")
     use_cache: Optional[bool] = Field(True, description="Use cached responses if available")