atakan Claude Sonnet 5 commited on
Commit
ad33173
·
1 Parent(s): 6be46a5

fix: Increase llama.cpp context window to fit the system prompt + tool schemas

Browse files

The system prompt plus all 29 registered tool JSON schemas alone total
~6,436 tokens, which already exceeded the previous n_ctx=4096 before a
single user message or generated token was added. Every request was
failing immediately with "Requested tokens exceed context window".
Bumped to 16384 to leave headroom for multi-step tool-calling and
generation.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

Files changed (1) hide show
  1. controlai_agent/orchestrator.py +2 -2
controlai_agent/orchestrator.py CHANGED
@@ -171,7 +171,7 @@ class ControlAIAgent:
171
  self.llama_model = llama_cpp.Llama(
172
  model_path=str(model_path),
173
  n_gpu_layers=-1, # Offload all layers to Metal / CUDA GPU
174
- n_ctx=4096,
175
  verbose=False,
176
  )
177
  self.hf_tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-3B-Instruct", trust_remote_code=True)
@@ -196,7 +196,7 @@ class ControlAIAgent:
196
  self.llama_model = llama_cpp.Llama.from_pretrained(
197
  repo_id=gguf_repo,
198
  filename=gguf_filename,
199
- n_ctx=4096,
200
  n_threads=threads,
201
  verbose=False,
202
  )
 
171
  self.llama_model = llama_cpp.Llama(
172
  model_path=str(model_path),
173
  n_gpu_layers=-1, # Offload all layers to Metal / CUDA GPU
174
+ n_ctx=16384,
175
  verbose=False,
176
  )
177
  self.hf_tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-3B-Instruct", trust_remote_code=True)
 
196
  self.llama_model = llama_cpp.Llama.from_pretrained(
197
  repo_id=gguf_repo,
198
  filename=gguf_filename,
199
+ n_ctx=16384,
200
  n_threads=threads,
201
  verbose=False,
202
  )